diff --git a/.backportrc.json b/.backportrc.json index db4418e180190..7f7365d78b124 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,10 +1,10 @@ { "upstream" : "elastic/elasticsearch", - "targetBranchChoices" : [ "main", "8.x", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], + "targetBranchChoices" : [ "main", "8.x", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], "targetPRLabels" : [ "backport" ], "branchLabelMapping" : { - "^v9.0.0$" : "main", + "^v9.1.0$" : "main", "^v8.18.0$" : "8.x", "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" } -} \ No newline at end of file +} diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f25092bc6d42f..6bb549df8a1fb 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -44,6 +44,9 @@ export GRADLE_BUILD_CACHE_USERNAME GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache) export GRADLE_BUILD_CACHE_PASSWORD +DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)" +export DEVELOCITY_ACCESS_KEY + BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token) export BUILDKITE_API_TOKEN @@ -91,6 +94,14 @@ if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then fi fi +# Authenticate to the Docker Hub public read-only registry +if which docker > /dev/null 2>&1; then + DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)" + DOCKERHUB_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)" + + echo "$DOCKERHUB_REGISTRY_PASSWORD" | docker login --username "$DOCKERHUB_REGISTRY_USERNAME" --password-stdin docker.io +fi + if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then # Run in the background, while the job continues nohup .buildkite/scripts/setup-monitoring.sh /dev/null 2>&1 & diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index a4b2b02919614..dd1e6c181f530 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -56,7 +56,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.28", "8.16.4", "8.17.2", "8.18.0"] + BWC_VERSION: ["7.17.29", "8.16.7", "8.17.5", "8.18.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-fwc.template.yml b/.buildkite/pipelines/periodic-fwc.template.yml new file mode 100644 index 0000000000000..2ce3b6673543c --- /dev/null +++ b/.buildkite/pipelines/periodic-fwc.template.yml @@ -0,0 +1,14 @@ +steps: + - label: $FWC_VERSION / fwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + matrix: + setup: + FWC_VERSION: $FWC_LIST + env: + FWC_VERSION: $FWC_VERSION diff --git a/.buildkite/pipelines/periodic-fwc.yml b/.buildkite/pipelines/periodic-fwc.yml new file mode 100644 index 0000000000000..1d47546474e47 --- /dev/null +++ b/.buildkite/pipelines/periodic-fwc.yml @@ -0,0 +1,15 @@ +# This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml +steps: + - label: $FWC_VERSION / fwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + matrix: + setup: + FWC_VERSION: [] + env: + FWC_VERSION: $FWC_VERSION diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index 38e4a71ccea93..1106ab0280532 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -43,7 +43,6 @@ steps: matrix: setup: image: - - windows-2016 - windows-2019 - windows-2022 agents: diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index c7fe5c5c724a0..99849b25c77da 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -307,8 +307,8 @@ steps: env: BWC_VERSION: 7.16.3 - - label: "{{matrix.image}} / 7.17.28 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.28 + - label: "{{matrix.image}} / 7.17.29 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.29 timeout_in_minutes: 300 matrix: setup: @@ -321,7 +321,7 @@ steps: machineType: custom-16-32768 buildDirectory: /dev/shm/bk env: - BWC_VERSION: 7.17.28 + BWC_VERSION: 7.17.29 - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1 @@ -579,8 +579,8 @@ steps: env: BWC_VERSION: 8.15.5 - - label: "{{matrix.image}} / 8.16.4 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.4 + - label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7 timeout_in_minutes: 300 matrix: setup: @@ -593,10 +593,10 @@ steps: machineType: custom-16-32768 buildDirectory: /dev/shm/bk env: - BWC_VERSION: 8.16.4 + BWC_VERSION: 8.16.7 - - label: "{{matrix.image}} / 8.17.2 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.2 + - label: "{{matrix.image}} / 8.17.5 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.5 timeout_in_minutes: 300 matrix: setup: @@ -609,7 +609,7 @@ steps: machineType: custom-16-32768 buildDirectory: /dev/shm/bk env: - BWC_VERSION: 8.17.2 + BWC_VERSION: 8.17.5 - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0 @@ -636,7 +636,6 @@ steps: matrix: setup: image: - - windows-2016 - windows-2019 - windows-2022 agents: diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index e930e53b0ccd8..91f33ff2b881f 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -42,7 +42,6 @@ steps: matrix: setup: image: - - windows-2016 - windows-2019 - windows-2022 GRADLE_TASK: diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 16e561193b250..6bdd016cd1d22 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -325,8 +325,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 7.17.28 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.28#bwcTest + - label: 7.17.29 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.29#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -335,7 +335,7 @@ steps: buildDirectory: /dev/shm/bk preemptible: true env: - BWC_VERSION: 7.17.28 + BWC_VERSION: 7.17.29 retry: automatic: - exit_status: "-1" @@ -648,8 +648,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.16.4 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.4#bwcTest + - label: 8.16.7 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -658,7 +658,7 @@ steps: buildDirectory: /dev/shm/bk preemptible: true env: - BWC_VERSION: 8.16.4 + BWC_VERSION: 8.16.7 retry: automatic: - exit_status: "-1" @@ -667,8 +667,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.17.2 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.2#bwcTest + - label: 8.17.5 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.5#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -677,7 +677,7 @@ steps: buildDirectory: /dev/shm/bk preemptible: true env: - BWC_VERSION: 8.17.2 + BWC_VERSION: 8.17.5 retry: automatic: - exit_status: "-1" @@ -771,7 +771,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.28", "8.16.4", "8.17.2", "8.18.0"] + BWC_VERSION: ["7.17.29", "8.16.7", "8.17.5", "8.18.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -819,7 +819,7 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.28", "8.16.4", "8.17.2", "8.18.0"] + BWC_VERSION: ["7.17.29", "8.16.7", "8.17.5", "8.18.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-windows.yml b/.buildkite/pipelines/pull-request/packaging-tests-windows.yml index 651a82982460f..6fd5cb90f1f00 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-windows.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-windows.yml @@ -10,7 +10,6 @@ steps: matrix: setup: image: - - windows-2016 - windows-2019 - windows-2022 PACKAGING_TASK: diff --git a/.buildkite/pipelines/pull-request/part-1-fips.yml b/.buildkite/pipelines/pull-request/part-1-fips.yml index 42f930c1bde9a..20518200cd9b8 100644 --- a/.buildkite/pipelines/pull-request/part-1-fips.yml +++ b/.buildkite/pipelines/pull-request/part-1-fips.yml @@ -1,5 +1,7 @@ config: - allow-labels: "Team:Security" + allow-labels: + - Team:Security + - test-fips steps: - label: part-1-fips command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart1 diff --git a/.buildkite/pipelines/pull-request/part-2-fips.yml b/.buildkite/pipelines/pull-request/part-2-fips.yml index 6a3647ceb50ae..aa555cfd2b24f 100644 --- a/.buildkite/pipelines/pull-request/part-2-fips.yml +++ b/.buildkite/pipelines/pull-request/part-2-fips.yml @@ -1,5 +1,7 @@ config: - allow-labels: "Team:Security" + allow-labels: + - Team:Security + - test-fips steps: - label: part-2-fips command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart2 diff --git a/.buildkite/pipelines/pull-request/part-3-fips.yml b/.buildkite/pipelines/pull-request/part-3-fips.yml index cee3ea153acb9..a95d7d50ac9d9 100644 --- a/.buildkite/pipelines/pull-request/part-3-fips.yml +++ b/.buildkite/pipelines/pull-request/part-3-fips.yml @@ -1,5 +1,7 @@ config: - allow-labels: "Team:Security" + allow-labels: + - Team:Security + - test-fips steps: - label: part-3-fips command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart3 diff --git a/.buildkite/pipelines/pull-request/part-4-fips.yml b/.buildkite/pipelines/pull-request/part-4-fips.yml index 5c020117ff00f..d0a580817046c 100644 --- a/.buildkite/pipelines/pull-request/part-4-fips.yml +++ b/.buildkite/pipelines/pull-request/part-4-fips.yml @@ -1,5 +1,7 @@ config: - allow-labels: "Team:Security" + allow-labels: + - Team:Security + - test-fips steps: - label: part-4-fips command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4 diff --git a/.buildkite/pipelines/pull-request/part-5-fips.yml b/.buildkite/pipelines/pull-request/part-5-fips.yml index 4e193ac751086..3c73d396cd4f1 100644 --- a/.buildkite/pipelines/pull-request/part-5-fips.yml +++ b/.buildkite/pipelines/pull-request/part-5-fips.yml @@ -1,5 +1,7 @@ config: - allow-labels: "Team:Security" + allow-labels: + - Team:Security + - test-fips steps: - label: part-5-fips command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart5 diff --git a/.buildkite/scripts/branches.sh b/.buildkite/scripts/branches.sh index 886fa59e4d02c..8e909b362ef2e 100755 --- a/.buildkite/scripts/branches.sh +++ b/.buildkite/scripts/branches.sh @@ -2,3 +2,7 @@ # This determines which branches will have pipelines triggered periodically, for dra workflows. BRANCHES=( $(cat branches.json | jq -r '.branches[].branch') ) + +# Sort them to make ordering predictable +IFS=$'\n' BRANCHES=($(sort <<<"${BRANCHES[*]}")) +unset IFS diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index cc10a5ae41861..30e13386f3088 100755 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -46,4 +46,15 @@ EOF branch: "$BRANCH" commit: "$LAST_GOOD_COMMIT" EOF +# Include forward compatibility tests only for the bugfix branch +if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then + cat < artifactoryCredentials = vault.logical() - .read("${vaultPathPrefix}/artifactory.elstc.co") - .getData() - logger.info("Using elastic artifactory repos") - Closure configCache = { - return { - name "artifactory-gradle-release" - url "https://artifactory.elstc.co/artifactory/gradle-release" - credentials { - username artifactoryCredentials.get("username") - password artifactoryCredentials.get("token") - } - } - } - settingsEvaluated { settings -> - settings.pluginManagement { - repositories { - maven configCache() - } - } - } - projectsLoaded { - allprojects { - buildscript { - repositories { - maven configCache() - } - } - repositories { - maven configCache() - } - } - } -} +final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url') +final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false')) gradle.settingsEvaluated { settings -> settings.pluginManager.withPlugin("com.gradle.develocity") { settings.develocity { - server = 'https://gradle-enterprise.elastic.co' + server = "https://gradle-enterprise.elastic.co" } - } -} - - -final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url') -final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false')) - -if (buildCacheUrl) { - final Map buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical() - .read("${vaultPathPrefix}/gradle-build-cache") - .getData() - gradle.settingsEvaluated { settings -> - settings.buildCache { - local { - // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty - enabled = false - } - remote(HttpBuildCache) { - url = buildCacheUrl - push = buildCachePush - credentials { - username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username") - password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password") + if (buildCacheUrl) { + settings.buildCache { + local { + // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty + enabled = false + } + remote(settings.develocity.buildCache) { + enabled = true + push = buildCachePush } } } } } - diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 03a74169a4f98..47f682fc44965 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - - "7.17.28" - - "8.16.4" - - "8.17.2" + - "7.17.29" + - "8.16.7" + - "8.17.5" - "8.18.0" diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java index 652defa7b39cd..056f7810677a2 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java @@ -60,6 +60,9 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +/** + * Benchmark for many different kinds of aggregator and groupings. + */ @Warmup(iterations = 5) @Measurement(iterations = 7) @BenchmarkMode(Mode.AverageTime) diff --git a/build-conventions/settings.gradle b/build-conventions/settings.gradle index 3256de7d56286..6d0b8adba42d3 100644 --- a/build-conventions/settings.gradle +++ b/build-conventions/settings.gradle @@ -6,6 +6,11 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ + +plugins { + id "com.gradle.develocity" version "3.19.2" +} + rootProject.name = 'build-conventions' dependencyResolutionManagement { diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java index 58b183fac3155..48465cb08cc79 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java @@ -15,6 +15,7 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Transformer; +import org.gradle.api.invocation.Gradle; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.Copy; @@ -38,6 +39,15 @@ public class EclipseConventionPlugin implements Plugin { @Override public void apply(Project project) { project.getPlugins().apply(EclipsePlugin.class); + Gradle gradle = project.getGradle(); + + boolean isEclipse = project.getProviders().systemProperty("eclipse.launcher").isPresent() || // Gradle launched from Eclipse + project.getProviders().systemProperty("eclipse.application").isPresent() || // Gradle launched from the Eclipse compiler server + gradle.getStartParameter().getTaskNames().contains("eclipse") || // Gradle launched from the command line to do eclipse stuff + gradle.getStartParameter().getTaskNames().contains("cleanEclipse"); + // for eclipse ide specific hacks... + project.getExtensions().add("isEclipse", isEclipse); + EclipseModel eclipseModel = project.getExtensions().getByType(EclipseModel.class); EclipseProject eclipseProject = eclipseModel.getProject(); diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index f2a02645f8c09..c04ba9b90d5e7 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -282,8 +282,6 @@ dependencies { api "org.elasticsearch:build-conventions:$version" api "org.elasticsearch.gradle:build-tools:$version" - // same version as http client transitive dep - api buildLibs.commons.codec api buildLibs.apache.compress api buildLibs.nebula.info api buildLibs.apache.rat @@ -303,8 +301,8 @@ dependencies { api buildLibs.antlrst4 api buildLibs.asm api buildLibs.asm.tree - api buildLibs.httpclient - api buildLibs.httpcore + api buildLibs.httpclient5 + api buildLibs.httpcore5 compileOnly buildLibs.checkstyle compileOnly buildLibs.reflections diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index e712035eabc7b..2a6e21b2ba89a 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip +distributionSha256Sum=fba8464465835e74f7270bbf43d6d8a8d7709ab0a43ce1aa3323f73e9aa0c612 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle index 8c88d36046768..ba38a18e19207 100644 --- a/build-tools-internal/settings.gradle +++ b/build-tools-internal/settings.gradle @@ -5,7 +5,11 @@ pluginManagement { } includeBuild "../build-conventions" - includeBuild "../build-tools" + includeBuild "../build-tools" +} + +plugins { + id "com.gradle.develocity" version "3.19.2" } dependencyResolutionManagement { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy index d93cc4b1873f2..3b149f5dcff66 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy @@ -20,6 +20,22 @@ abstract class AbstractGradleInternalPluginFuncTest extends AbstractJavaGradleFu plugins { id 'elasticsearch.java-toolchain' } + + toolchainManagement { + jvm { + javaRepositories { + repository('bundledOracleOpendJdk') { + resolverClass = org.elasticsearch.gradle.internal.toolchain.OracleOpenJdkToolchainResolver + } + repository('adoptiumJdks') { + resolverClass = org.elasticsearch.gradle.internal.toolchain.AdoptiumJdkToolchainResolver + } + repository('archivedOracleJdks') { + resolverClass = org.elasticsearch.gradle.internal.toolchain.ArchivedOracleJdkToolchainResolver + } + } + } + } """ + settingsFile.text buildFile << """ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy index 1396272026abc..150f2b1130159 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy @@ -13,6 +13,27 @@ package org.elasticsearch.gradle.fixtures; abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest { def setup() { + settingsFile.text = """ + plugins { + id 'elasticsearch.java-toolchain' + } + + toolchainManagement { + jvm { + javaRepositories { + repository('bundledOracleOpendJdk') { + resolverClass = org.elasticsearch.gradle.internal.toolchain.OracleOpenJdkToolchainResolver + } + repository('adoptiumJdks') { + resolverClass = org.elasticsearch.gradle.internal.toolchain.AdoptiumJdkToolchainResolver + } + repository('archivedOracleJdks') { + resolverClass = org.elasticsearch.gradle.internal.toolchain.ArchivedOracleJdkToolchainResolver + } + } + } + } + """ + settingsFile.text subProject(":test:framework") << "apply plugin: 'elasticsearch.java'" subProject(":test:test-clusters") << "apply plugin: 'elasticsearch.java'" subProject(":test:yaml-rest-runner") << "apply plugin: 'elasticsearch.java'" diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy index 733742653e4a3..e6ddad01933fb 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy @@ -14,9 +14,55 @@ import org.gradle.testkit.runner.TaskOutcome class ElasticsearchTestBasePluginFuncTest extends AbstractGradleFuncTest { - def setup() { - // see https://github.com/gradle/gradle/issues/24172 - configurationCacheCompatible = false + def "can disable assertions via cmdline param"() { + given: + file("src/test/java/acme/SomeTests.java").text = """ + public class SomeTests { + @org.junit.Test + public void testAsserts() { + assert false; + } + } + """ + buildFile.text = """ + plugins { + id 'java' + id 'elasticsearch.test-base' + } + + repositories { + mavenCentral() + } + + dependencies { + testImplementation 'junit:junit:4.12' + } + """ + + when: + def result = gradleRunner("test").buildAndFail() + then: + result.task(':test').outcome == TaskOutcome.FAILED + + when: + result = gradleRunner("test", "-Dtests.asserts=false").build() + then: + result.task(':test').outcome == TaskOutcome.SUCCESS + + when: + result = gradleRunner("test", "-Dtests.jvm.argline=-da").build() + then: + result.task(':test').outcome == TaskOutcome.SUCCESS + + when: + result = gradleRunner("test", "-Dtests.jvm.argline=-disableassertions").build() + then: + result.task(':test').outcome == TaskOutcome.SUCCESS + + when: + result = gradleRunner("test", "-Dtests.asserts=false", "-Dtests.jvm.argline=-da").build() + then: + result.task(':test').outcome == TaskOutcome.SUCCESS } def "can configure nonInputProperties for test tasks"() { diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle index c671c18cad030..cf74de2286583 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle @@ -34,7 +34,7 @@ subprojects { } def calculateBranchVersion() { - File f = rootProject.file(".git/refs/heads/origin") + File f = layout.settingsDirectory.file(".git/refs/heads/origin").asFile def branchName = f.list()?.first().trim() return branchName + ".1" diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index bc6d6f8d48d33..30060cf9bb40f 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -28,7 +28,9 @@ develocity { // Automatically publish scans from Elasticsearch CI if (onCI) { publishing.onlyIf { true } - server = 'https://gradle-enterprise.elastic.co' + if(server.isPresent() == false) { + server = 'https://gradle-enterprise.elastic.co' + } } else if( server.isPresent() == false) { publishing.onlyIf { false } } @@ -127,6 +129,9 @@ develocity { } } else { tag 'LOCAL' + if (System.getProperty('idea.active') == 'true') { + tag 'IDEA' + } } } } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.fwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.fwc-test.gradle new file mode 100644 index 0000000000000..51301b405e514 --- /dev/null +++ b/build-tools-internal/src/main/groovy/elasticsearch.fwc-test.gradle @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +def fwcVersions = buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor } +def previousMinorSnapshot = buildParams.bwcVersions.unreleased.find { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor - 1 } + +fwcVersions.each { fwcVersion -> + tasks.register("v${fwcVersion}#fwcTest", StandaloneRestIntegTestTask) { + usesBwcDistribution(previousMinorSnapshot) + usesBwcDistribution(fwcVersion) + systemProperty("tests.old_cluster_version", previousMinorSnapshot) + systemProperty("tests.new_cluster_version", fwcVersion) + nonInputProperties.systemProperty 'tests.fwc', 'true' + } +} + +gradle.taskGraph.whenReady { graph -> + if (graph.allTasks.any { it.name.endsWith('#fwcTest') } && Boolean.parseBoolean(System.getProperty("tests.bwc.snapshot", "true"))) { + throw new GradleException("Running forward compatibility tests requires passing `-Dtests.bwc.snapshot=false`.") + } + + if (graph.allTasks.any { it.name.endsWith('#fwcTest') } && graph.allTasks.any { it.name.endsWith('#bwcTest') }) { + throw new GradleException("Backward compatibility and forward compatibility tests cannot be executed in the same build.") + } +} diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 671c79d9ed966..a0521f329b52b 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -132,8 +132,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { tasks.register('buildDependencyArtifacts') { group = 'ide' description = 'Builds artifacts needed as dependency for IDE modules' - dependsOn([':plugins:repository-hdfs:hadoop-client-api:jar', - ':x-pack:plugin:esql:compute:ann:jar', + dependsOn([':x-pack:plugin:esql:compute:ann:jar', ':x-pack:plugin:esql:compute:gen:jar', ':server:generateModulesList', ':server:generatePluginsList', @@ -144,7 +143,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { // this path is produced by the extractLibs task above String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform") - + def enableIdeaCC = providers.gradleProperty("org.elasticsearch.idea-configuration-cache").getOrElse("true").toBoolean() idea { project { vcs = 'Git' @@ -171,6 +170,11 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { } } runConfigurations { + defaults(org.jetbrains.gradle.ext.Gradle) { + scriptParameters = enableIdeaCC ? [ + '--configuration-cache' + ].join(' ') : '' + } defaults(JUnit) { vmParameters = [ '-ea', diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index 49887dac5b6fd..b66eb085b62f8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -17,10 +17,7 @@ import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; -import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; -import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; -import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -39,7 +36,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class); - boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).isCi(); + boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds // TODO remove once we removed default dependencies from PluginBuildPlugin @@ -81,29 +78,6 @@ public void doCall() { if (isModule == false || isXPackModule) { addNoticeGeneration(project, extension); } - project.afterEvaluate(p -> { - @SuppressWarnings("unchecked") - NamedDomainObjectContainer testClusters = (NamedDomainObjectContainer) project - .getExtensions() - .getByName(TestClustersPlugin.EXTENSION_NAME); - p.getExtensions().getByType(PluginPropertiesExtension.class).getExtendedPlugins().forEach(pluginName -> { - // Auto add any dependent modules - findModulePath(project, pluginName).ifPresent( - path -> testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.module(path)) - ); - }); - }); - } - - Optional findModulePath(Project project, String pluginName) { - return project.getRootProject() - .getAllprojects() - .stream() - .filter(p -> GradleUtils.isModuleProject(p.getPath())) - .filter(p -> p.getPlugins().hasPlugin(PluginBuildPlugin.class)) - .filter(p -> p.getExtensions().getByType(PluginPropertiesExtension.class).getName().equals(pluginName)) - .findFirst() - .map(Project::getPath); } /** diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index d7bf839817e12..41652ebe8a0df 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -138,7 +138,14 @@ private static TaskProvider createRunBwcGradleTask( loggedExec.args("-DisCI"); } - loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED"); + loggedExec.args("-Dscan.tag.NESTED"); + + if (System.getProperty("tests.bwc.snapshot", "true").equals("false")) { + loggedExec.args("-Dbuild.snapshot=false", "-Dlicense.key=x-pack/plugin/core/snapshot.key"); + } else { + loggedExec.args("-Dbuild.snapshot=true"); + } + final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel(); List nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG); if (nonDefaultLogLevels.contains(logLevel)) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index ae9decf668f04..7ea34f1eb93b8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -24,7 +24,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bfdeddb33330a281950c2a54adef991dbbe6a42832bc505d13b11beaf50ae73f", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:5ba5defb8c55ea6e2d4a697ca82da534c21c6328dfa903fe460c762720875794", "-wolfi", "apk" ), diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java index b1207a2f5161d..065a57f801c9e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java @@ -110,6 +110,8 @@ private List resolveProjectLogs(File projectDir) { projectDirFiles.include("**/build/testrun/*/temp/**"); projectDirFiles.include("**/build/**/hs_err_pid*.log"); projectDirFiles.include("**/build/**/replay_pid*.log"); + // core dump files are in the working directory of the installation, which is not project specific + projectDirFiles.include("distribution/**/build/install/*/core.*"); projectDirFiles.exclude("**/build/testclusters/**/data/**"); projectDirFiles.exclude("**/build/testclusters/**/distro/**"); projectDirFiles.exclude("**/build/testclusters/**/repo/**"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index ee0eb3f6eb2bf..70f6cecb8e725 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -58,7 +58,6 @@ public class ElasticsearchJavaBasePlugin implements Plugin { @Override public void apply(Project project) { - project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); // make sure the global build info plugin is applied to the root project project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); @@ -162,7 +161,7 @@ public void configureCompile(Project project) { compileTask.getConventionMapping().map("sourceCompatibility", () -> java.getSourceCompatibility().toString()); compileTask.getConventionMapping().map("targetCompatibility", () -> java.getTargetCompatibility().toString()); compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask)); - compileOptions.setIncremental(buildParams.isCi() == false); + compileOptions.setIncremental(buildParams.getCi() == false); }); // also apply release flag to groovy, which is used in build-tools project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 240b55dedf7ce..fa75a2011ab90 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -137,10 +137,15 @@ public void execute(Task t) { test.jvmArgs((Object[]) argline.split(" ")); } - if (Util.getBooleanProperty("tests.asserts", true)) { - test.jvmArgs("-ea", "-esa"); + // Check if "tests.asserts" is false or "tests.jvm.argline" contains the "-da" flag. + boolean disableAssertions = Util.getBooleanProperty("tests.asserts", true) == false + || (argline != null && (argline.contains("-da"))) + || (argline != null && (argline.contains("-disableassertions"))); + + if (disableAssertions) { + System.out.println("disable assertions"); + test.setEnableAssertions(false); } - Map sysprops = Map.of( "java.awt.headless", "true", diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java index 805a91b03ee4e..fa620c736daa1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java @@ -19,6 +19,8 @@ import org.gradle.api.Task; import org.gradle.api.file.ArchiveOperations; import org.gradle.api.plugins.BasePlugin; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Copy; import org.gradle.api.tasks.TaskProvider; @@ -103,22 +105,26 @@ private static TaskProvider registerCheckMlCppNoticeTask( ) { TaskProvider checkMlCppNoticeTask = project.getTasks().register("checkMlCppNotice", task -> { task.dependsOn(checkExtraction); + final Provider noticePath = checkExtraction.map( + c -> c.getDestinationDir() + .toPath() + .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt") + ); + ListProperty expectedMlLicenses = extension.expectedMlLicenses; task.doLast(new Action() { @Override public void execute(Task task) { // this is just a small sample from the C++ notices, // the idea being that if we've added these lines we've probably added all the required lines - final List expectedLines = extension.expectedMlLicenses.get(); - final Path noticePath = checkExtraction.get() - .getDestinationDir() - .toPath() - .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt"); + final List expectedLines = expectedMlLicenses.get(); final List actualLines; try { - actualLines = Files.readAllLines(noticePath); + actualLines = Files.readAllLines(noticePath.get()); for (final String expectedLine : expectedLines) { if (actualLines.contains(expectedLine) == false) { - throw new GradleException("expected [" + noticePath + " to contain [" + expectedLine + "] but it did not"); + throw new GradleException( + "expected [" + noticePath.get() + " to contain [" + expectedLine + "] but it did not" + ); } } } catch (IOException ioException) { @@ -133,16 +139,12 @@ public void execute(Task task) { private TaskProvider registerCheckNoticeTask(Project project, TaskProvider checkExtraction) { return project.getTasks().register("checkNotice", task -> { task.dependsOn(checkExtraction); - task.doLast(new Action() { - @Override - public void execute(Task task) { - final List noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2024 Elasticsearch"); - final Path noticePath = checkExtraction.get() - .getDestinationDir() - .toPath() - .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt"); - assertLinesInFile(noticePath, noticeLines); - } + var noticePath = checkExtraction.map( + copy -> copy.getDestinationDir().toPath().resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt") + ); + task.doLast(t -> { + final List noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2024 Elasticsearch"); + assertLinesInFile(noticePath.get(), noticeLines); }); }); } @@ -150,26 +152,24 @@ public void execute(Task task) { private TaskProvider registerCheckLicenseTask(Project project, TaskProvider checkExtraction) { TaskProvider checkLicense = project.getTasks().register("checkLicense", task -> { task.dependsOn(checkExtraction); - task.doLast(new Action() { - @Override - public void execute(Task task) { - String licenseFilename = null; - if (project.getName().contains("oss-") || project.getName().equals("integ-test-zip")) { - licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; - } else { - licenseFilename = "ELASTIC-LICENSE-2.0.txt"; - } - final List licenseLines; - try { - licenseLines = Files.readAllLines(project.getRootDir().toPath().resolve("licenses/" + licenseFilename)); - final Path licensePath = checkExtraction.get() - .getDestinationDir() - .toPath() - .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt"); - assertLinesInFile(licensePath, licenseLines); - } catch (IOException ioException) { - ioException.printStackTrace(); - } + String projectName = project.getName(); + Provider licensePathProvider = checkExtraction.map( + copy -> copy.getDestinationDir().toPath().resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt") + ); + File rootDir = project.getLayout().getSettingsDirectory().getAsFile(); + task.doLast(t -> { + String licenseFilename = null; + if (projectName.contains("oss-") || projectName.equals("integ-test-zip")) { + licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; + } else { + licenseFilename = "ELASTIC-LICENSE-2.0.txt"; + } + final List licenseLines; + try { + licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename)); + assertLinesInFile(licensePathProvider.get(), licenseLines); + } catch (IOException ioException) { + ioException.printStackTrace(); } }); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 28ce3df6c7815..2903b4d33c08b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -72,7 +72,7 @@ public void apply(Project project) { project.getPlugins().apply(JvmToolchainsPlugin.class); toolChainService = project.getExtensions().getByType(JavaToolchainService.class); var buildParams = loadBuildParams(project).get(); - Boolean isCi = buildParams.isCi(); + Boolean isCi = buildParams.getCi(); buildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { configureBwcProject( project.project(unreleasedVersion.gradleProjectPath()), @@ -336,8 +336,9 @@ static void createBuildBwcTask( String bwcTaskName = buildBwcTaskName(projectName); bwcSetupExtension.bwcTask(bwcTaskName, c -> { boolean useNativeExpanded = projectArtifact.expandedDistDir != null; + boolean isReleaseBuild = System.getProperty("tests.bwc.snapshot", "true").equals("false"); File expectedOutputFile = useNativeExpanded - ? new File(projectArtifact.expandedDistDir, "elasticsearch-" + bwcVersion.get() + "-SNAPSHOT") + ? new File(projectArtifact.expandedDistDir, "elasticsearch-" + bwcVersion.get() + (isReleaseBuild ? "" : "-SNAPSHOT")) : projectArtifact.distFile; c.getInputs().file(new File(project.getBuildDir(), "refspec")); if (useNativeExpanded) { @@ -345,7 +346,7 @@ static void createBuildBwcTask( } else { c.getOutputs().files(expectedOutputFile); } - c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> buildParams.isCi() == false); + c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> buildParams.getCi() == false); c.getArgs().add("-p"); c.getArgs().add(projectPath); c.getArgs().add(assembleTaskName); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java index c618fe6c2e1bf..f1b96016286df 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java @@ -30,7 +30,7 @@ public void apply(Project project) { TestClustersPlugin testClustersPlugin = project.getPlugins().apply(TestClustersPlugin.class); testClustersPlugin.setRuntimeJava(buildParams.getRuntimeJavaHome()); testClustersPlugin.setIsReleasedVersion( - version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.isSnapshotBuild() == false) + version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.getSnapshotBuild() == false) || buildParams.getBwcVersions().unreleasedInfo(version) == null ); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java index 9a85f44953b6d..190a33fd34d69 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java @@ -111,7 +111,11 @@ private String getPath() { List classNames() throws IOException { Pattern classEnding = Pattern.compile(".*\\.class$"); try (JarFile jf = new JarFile(this.path)) { - return jf.stream().map(ZipEntry::getName).filter(classEnding.asMatchPredicate()).collect(Collectors.toList()); + return jf.stream() + .map(ZipEntry::getName) + .filter(classEnding.asMatchPredicate()) + .filter(c -> c.startsWith("org/elasticsearch/logging/internal/") == false) + .collect(Collectors.toList()); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java index a14223b73018b..6baa2b09128f7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java @@ -72,12 +72,13 @@ public void apply(Project project) { var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class); var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true"); var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true"); + int minJavaVersion = Integer.parseInt(buildParams.getMinimumCompilerVersion().getMajorVersion()); // Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this. // Avoids an IntelliJ bug: // https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea if (isIdeaSync == false || ideaSourceSetsEnabled) { - List mainVersions = findSourceVersions(project); + List mainVersions = findSourceVersions(project, minJavaVersion); List mainSourceSets = new ArrayList<>(); mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME); List testSourceSets = new ArrayList<>(mainSourceSets); @@ -101,6 +102,7 @@ public void apply(Project project) { } private void configureMrjar(Project project) { + var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME); jarTask.configure(task -> { task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); }); }); @@ -216,7 +218,7 @@ private void createTestTask( project.getTasks().named("check").configure(checkTask -> checkTask.dependsOn(testTaskProvider)); } - private static List findSourceVersions(Project project) { + private static List findSourceVersions(Project project, int minJavaVersion) { var srcDir = project.getProjectDir().toPath().resolve("src"); List versions = new ArrayList<>(); try (var subdirStream = Files.list(srcDir)) { @@ -225,7 +227,23 @@ private static List findSourceVersions(Project project) { String sourcesetName = sourceSetPath.getFileName().toString(); Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName); if (sourcesetMatcher.matches()) { - versions.add(Integer.parseInt(sourcesetMatcher.group(1))); + int version = Integer.parseInt(sourcesetMatcher.group(1)); + if (version < minJavaVersion) { + // NOTE: We allow mainNN for the min java version so that incubating modules can be used without warnings. + // It is a workaround for https://bugs.openjdk.org/browse/JDK-8187591. Once min java is 22, we + // can use the SuppressWarnings("preview") in the code using incubating modules and this check + // can change to <= + throw new IllegalArgumentException( + "Found src dir '" + + sourcesetName + + "' for Java " + + version + + " but multi-release jar sourceset should have version " + + minJavaVersion + + " or greater" + ); + } + versions.add(version); } } } catch (IOException e) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java index 7e3e8bd458c92..4f3c4b3d94f68 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java @@ -29,56 +29,14 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag ListMultimap, String> map = ArrayListMultimap.create(1, 200); map.put(LegacyRestTestBasePlugin.class, ":docs"); map.put(LegacyRestTestBasePlugin.class, ":distribution:docker"); - map.put(LegacyRestTestBasePlugin.class, ":modules:lang-expression"); - map.put(LegacyRestTestBasePlugin.class, ":modules:lang-mustache"); - map.put(LegacyRestTestBasePlugin.class, ":modules:mapper-extras"); - map.put(LegacyRestTestBasePlugin.class, ":modules:parent-join"); - map.put(LegacyRestTestBasePlugin.class, ":modules:percolator"); - map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval"); map.put(LegacyRestTestBasePlugin.class, ":modules:reindex"); - map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url"); - map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-kuromoji"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-nori"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-phonetic"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-smartcn"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-stempel"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-ukrainian"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-azure-classic"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-annotated-text"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-murmur3"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:repository-hdfs"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb"); map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster"); map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster"); map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search"); map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters"); map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version"); map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy"); - map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http"); - map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled"); - map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies"); - map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-plugins"); - map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices"); - map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants"); - map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration"); - map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs"); - map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity"); - map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-error-query"); - map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-latency-simulating-directory"); - map.put(LegacyRestTestBasePlugin.class, ":test:yaml-rest-runner"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:core"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-constant-keyword"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-unsigned-long"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-version"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:wildcard"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade"); @@ -92,9 +50,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-plugins-ssl"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-security-with-mustache"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:xpack-prefix-rest-compat"); - map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-geoip:qa:file-based-update"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2:qa:amazon-ec2"); - map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce:qa:gce"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust"); @@ -103,12 +58,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:autoscaling:qa:rest"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:downgrade-to-basic-license"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:multi-cluster"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:non-compliant-license"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:rest"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:restart"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:security"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security"); @@ -119,7 +68,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:mixed-node"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet:qa:rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:identity-provider:qa:idp-rest-tests"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-cluster"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-node"); @@ -136,8 +84,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:tls-basic"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:multi-node"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:rolling-upgrade"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:multi-node"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:fs"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs"); @@ -150,12 +96,10 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:mixed-node"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:with-ssl"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:without-ssl"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:stack:qa:rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:text-structure:qa:text-structure-with-security"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-cluster-tests-with-security"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-node-tests"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:single-node-tests"); - map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile:qa:multi-cluster"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:rest"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:with-security"); return map; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java new file mode 100644 index 0000000000000..adb39368d8d24 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; + +import org.gradle.api.artifacts.transform.CacheableTransform; +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; +import org.jetbrains.annotations.NotNull; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.regex.Pattern; + +import static java.util.Map.entry; +import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; +import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; + +@CacheableTransform +public abstract class HdfsClassPatcher implements TransformAction { + + record JarPatchers(String artifactTag, Pattern artifactPattern, Map> jarPatchers) {} + + static final List allPatchers = List.of( + new JarPatchers( + "hadoop-common", + Pattern.compile("hadoop-common-(?!.*tests)"), + Map.ofEntries( + entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), + entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new), + entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new) + ) + ), + new JarPatchers( + "hadoop-client-api", + Pattern.compile("hadoop-client-api.*"), + Map.ofEntries( + entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), + entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new), + entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new), + entry("org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", SubjectGetSubjectPatcher::new) + ) + ) + ); + + interface Parameters extends TransformParameters { + @Input + @Optional + List getMatchingArtifacts(); + + void setMatchingArtifacts(List matchingArtifacts); + } + + @Classpath + @InputArtifact + public abstract Provider getInputArtifact(); + + @Override + public void transform(@NotNull TransformOutputs outputs) { + File inputFile = getInputArtifact().get().getAsFile(); + + List matchingArtifacts = getParameters().getMatchingArtifacts(); + List patchersToApply = allPatchers.stream() + .filter(jp -> matchingArtifacts.contains(jp.artifactTag()) && jp.artifactPattern().matcher(inputFile.getName()).find()) + .toList(); + if (patchersToApply.isEmpty()) { + outputs.file(getInputArtifact()); + } else { + patchersToApply.forEach(patchers -> { + System.out.println("Patching " + inputFile.getName()); + + Map> jarPatchers = new HashMap<>(patchers.jarPatchers()); + File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); + + patchJar(inputFile, outputFile, jarPatchers); + + if (jarPatchers.isEmpty() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "error patching [%s] with [%s]: the jar does not contain [%s]", + inputFile.getName(), + patchers.artifactPattern().toString(), + String.join(", ", jarPatchers.keySet()) + ) + ); + } + }); + } + } + + private static void patchJar(File inputFile, File outputFile, Map> jarPatchers) { + try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String entryName = entry.getName(); + // Add the entry to the new JAR file + jos.putNextEntry(new JarEntry(entryName)); + + Function classPatcher = jarPatchers.remove(entryName); + if (classPatcher != null) { + byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); + + ClassReader classReader = new ClassReader(classToPatch); + ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS); + classReader.accept(classPatcher.apply(classWriter), 0); + + jos.write(classWriter.toByteArray()); + } else { + // Read the entry's data and write it to the new JAR + try (InputStream is = jarFile.getInputStream(entry)) { + is.transferTo(jos); + } + } + jos.closeEntry(); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java similarity index 94% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java index e07a32cc294a5..7bc6a6c0d530f 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/MethodReplacement.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java similarity index 94% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java index 397b63e434ba2..ab63249f5c8e8 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShellPatcher.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java similarity index 97% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java index 1235b5af9002f..4efe48a3bf72d 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/ShutdownHookManagerPatcher.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java similarity index 98% rename from plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java index e7c0002d349ba..00ce45af918f0 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.hdfs.patch; +package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java index 7348181c4199c..7ec35ccd32e10 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java @@ -40,7 +40,7 @@ public void apply(Project project) { .getSharedServices() .registerIfAbsent(DOCKER_SUPPORT_SERVICE_NAME, DockerSupportService.class, spec -> spec.parameters(params -> { params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE)); - params.getIsCI().set(buildParams.isCi()); + params.getIsCI().set(buildParams.getCi()); })); // Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java index ef9055b3728d3..d6c8e38dac2c3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java @@ -54,11 +54,11 @@ public interface BuildParameterExtension { String getTestSeed(); - Boolean isCi(); + Boolean getCi(); Integer getDefaultParallel(); - Boolean isSnapshotBuild(); + Boolean getSnapshotBuild(); BwcVersions getBwcVersions(); @@ -66,5 +66,5 @@ public interface BuildParameterExtension { Random getRandom(); - Boolean isGraalVmRuntime(); + Boolean getGraalVmRuntime(); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java index 283c02428e4e6..760664f9fa025 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java @@ -42,7 +42,7 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx private final String testSeed; private final Boolean isCi; private final Integer defaultParallel; - private final Boolean isSnapshotBuild; + private final Boolean snapshotBuild; // not final for testing private Provider bwcVersions; @@ -81,7 +81,7 @@ public DefaultBuildParameterExtension( this.testSeed = testSeed; this.isCi = isCi; this.defaultParallel = defaultParallel; - this.isSnapshotBuild = isSnapshotBuild; + this.snapshotBuild = isSnapshotBuild; this.bwcVersions = cache(providers, bwcVersions); this.gitOrigin = gitOrigin; } @@ -183,7 +183,7 @@ public String getTestSeed() { } @Override - public Boolean isCi() { + public Boolean getCi() { return isCi; } @@ -193,8 +193,8 @@ public Integer getDefaultParallel() { } @Override - public Boolean isSnapshotBuild() { - return isSnapshotBuild; + public Boolean getSnapshotBuild() { + return snapshotBuild; } @Override @@ -208,7 +208,7 @@ public Random getRandom() { } @Override - public Boolean isGraalVmRuntime() { + public Boolean getGraalVmRuntime() { return runtimeJavaDetails.get().toLowerCase().contains("graalvm"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 442797775de2f..b18a7d553d683 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -61,6 +61,7 @@ import static org.gradle.api.JavaVersion.VERSION_21; import static org.gradle.api.JavaVersion.VERSION_22; import static org.gradle.api.JavaVersion.VERSION_23; +import static org.gradle.api.JavaVersion.VERSION_24; @CacheableTask public abstract class ThirdPartyAuditTask extends DefaultTask { @@ -346,8 +347,12 @@ private String runForbiddenAPIsCli() throws IOException { spec.setExecutable(javaHome.get() + "/bin/java"); } spec.classpath(getForbiddenAPIsClasspath(), classpath); - // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module. - if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) { + // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23/24 for now, and just the vector module. + if (isJavaVersion(VERSION_20) + || isJavaVersion(VERSION_21) + || isJavaVersion(VERSION_22) + || isJavaVersion(VERSION_23) + || isJavaVersion(VERSION_24)) { spec.jvmArgs("--add-modules", "jdk.incubator.vector"); } spec.jvmArgs("-Xmx1g"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java index a6ead34b11079..ebd316d7f042a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java @@ -15,6 +15,7 @@ import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.VariableDeclarator; +import com.github.javaparser.ast.expr.Expression; import com.github.javaparser.ast.expr.NameExpr; import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter; import com.google.common.annotations.VisibleForTesting; @@ -33,6 +34,7 @@ import java.util.Objects; import java.util.Optional; import java.util.TreeMap; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -51,6 +53,8 @@ public class UpdateVersionsTask extends AbstractVersionsTask { private boolean setCurrent; @Nullable private Version removeVersion; + @Nullable + private String addTransportVersion; @Inject public UpdateVersionsTask(BuildLayout layout) { @@ -62,6 +66,11 @@ public void addVersion(String version) { this.addVersion = Version.fromString(version); } + @Option(option = "add-transport-version", description = "Specifies transport version to add") + public void addTransportVersion(String transportVersion) { + this.addTransportVersion = transportVersion; + } + @Option(option = "set-current", description = "Set the 'current' constant to the new version") public void setCurrent(boolean setCurrent) { this.setCurrent = setCurrent; @@ -87,15 +96,18 @@ static Optional parseVersionField(CharSequence field) { @TaskAction public void executeTask() throws IOException { - if (addVersion == null && removeVersion == null) { + if (addVersion == null && removeVersion == null && addTransportVersion == null) { throw new IllegalArgumentException("No versions to add or remove specified"); } if (setCurrent && addVersion == null) { throw new IllegalArgumentException("No new version added to set as the current version"); } - if (Objects.equals(addVersion, removeVersion)) { + if (addVersion != null && removeVersion != null && Objects.equals(addVersion, removeVersion)) { throw new IllegalArgumentException("Same version specified to add and remove"); } + if (addTransportVersion != null && addTransportVersion.split(":").length != 2) { + throw new IllegalArgumentException("Transport version specified must be in the format ':'"); + } Path versionJava = rootDir.resolve(VERSION_FILE_PATH); CompilationUnit file = LexicalPreservingPrinter.setup(StaticJavaParser.parse(versionJava)); @@ -115,6 +127,18 @@ public void executeTask() throws IOException { modifiedFile = removed; } } + if (addTransportVersion != null) { + var constant = addTransportVersion.split(":")[0]; + var versionId = Integer.parseInt(addTransportVersion.split(":")[1]); + LOGGER.lifecycle("Adding transport version constant [{}] with id [{}]", constant, versionId); + + var transportVersionsFile = rootDir.resolve(TRANSPORT_VERSIONS_FILE_PATH); + var transportVersions = LexicalPreservingPrinter.setup(StaticJavaParser.parse(transportVersionsFile)); + var modified = addTransportVersionConstant(transportVersions, constant, versionId); + if (modified.isPresent()) { + writeOutNewContents(transportVersionsFile, modified.get()); + } + } if (modifiedFile.isPresent()) { writeOutNewContents(versionJava, modifiedFile.get()); @@ -161,6 +185,51 @@ static Optional addVersionConstant(CompilationUnit versionJava, return Optional.of(versionJava); } + @VisibleForTesting + static Optional addTransportVersionConstant(CompilationUnit transportVersions, String constant, int versionId) { + ClassOrInterfaceDeclaration transportVersionsClass = transportVersions.getClassByName("TransportVersions").get(); + if (transportVersionsClass.getFieldByName(constant).isPresent()) { + LOGGER.lifecycle("New transport version constant [{}] already present, skipping", constant); + return Optional.empty(); + } + + TreeMap versions = transportVersionsClass.getFields() + .stream() + .filter(f -> f.getElementType().asString().equals("TransportVersion")) + .filter( + f -> f.getVariables().stream().limit(1).allMatch(v -> v.getInitializer().filter(Expression::isMethodCallExpr).isPresent()) + ) + .filter(f -> f.getVariable(0).getInitializer().get().asMethodCallExpr().getNameAsString().endsWith("def")) + .collect( + Collectors.toMap( + f -> f.getVariable(0) + .getInitializer() + .get() + .asMethodCallExpr() + .getArgument(0) + .asIntegerLiteralExpr() + .asNumber() + .intValue(), + Function.identity(), + (f1, f2) -> { + throw new IllegalStateException("Duplicate version constant " + f1); + }, + TreeMap::new + ) + ); + + // find the version this should be inserted after + Map.Entry previousVersion = versions.lowerEntry(versionId); + if (previousVersion == null) { + throw new IllegalStateException(String.format("Could not find previous version to [%s]", versionId)); + } + + FieldDeclaration newTransportVersion = createNewTransportVersionConstant(previousVersion.getValue(), constant, versionId); + transportVersionsClass.getMembers().addAfter(newTransportVersion, previousVersion.getValue()); + + return Optional.of(transportVersions); + } + private static FieldDeclaration createNewVersionConstant(FieldDeclaration lastVersion, String newName, String newExpr) { return new FieldDeclaration( new NodeList<>(lastVersion.getModifiers()), @@ -172,6 +241,29 @@ private static FieldDeclaration createNewVersionConstant(FieldDeclaration lastVe ); } + private static FieldDeclaration createNewTransportVersionConstant(FieldDeclaration lastVersion, String newName, int newId) { + return new FieldDeclaration( + new NodeList<>(lastVersion.getModifiers()), + new VariableDeclarator( + lastVersion.getCommonType(), + newName, + StaticJavaParser.parseExpression(String.format("def(%s)", formatTransportVersionId(newId))) + ) + ); + } + + private static String formatTransportVersionId(int id) { + String idString = Integer.toString(id); + + return new StringBuilder(idString.substring(idString.length() - 2, idString.length())).insert(0, "_") + .insert(0, idString.substring(idString.length() - 3, idString.length() - 2)) + .insert(0, "_") + .insert(0, idString.substring(idString.length() - 6, idString.length() - 3)) + .insert(0, "_") + .insert(0, idString.substring(0, idString.length() - 6)) + .toString(); + } + @VisibleForTesting static Optional removeVersionConstant(CompilationUnit versionJava, Version version) { String removeFieldName = toVersionField(version); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index 58b9c594a68bc..d908da1d2b4c0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -9,12 +9,14 @@ package org.elasticsearch.gradle.internal.snyk; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.FileEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; +import org.apache.hc.client5.http.classic.methods.HttpPut; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.FileEntity; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.file.RegularFileProperty; @@ -61,16 +63,16 @@ void upload() { HttpPut putRequest = new HttpPut(endpoint); putRequest.addHeader("Authorization", "token " + token.get()); putRequest.addHeader("Content-Type", "application/json"); - putRequest.setEntity(new FileEntity(inputFile.getAsFile().get())); + putRequest.setEntity(new FileEntity(inputFile.getAsFile().get(), ContentType.APPLICATION_JSON)); response = client.execute(putRequest); - int statusCode = response.getStatusLine().getStatusCode(); + int statusCode = response.getCode(); String responseString = EntityUtils.toString(response.getEntity()); getLogger().info("Snyk API call response status: " + statusCode); if (statusCode != HttpURLConnection.HTTP_CREATED) { throw new GradleException("Uploading Snyk Graph failed with http code " + statusCode + ": " + responseString); } getLogger().info(responseString); - } catch (IOException e) { + } catch (IOException | ParseException e) { throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java index c13a5f0e4d30d..ea6e8f0f1b01e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java @@ -51,7 +51,7 @@ public void apply(Project project) { } // Don't fail when all tests are ignored when running in CI - filter.setFailOnNoMatchingTests(buildParams.isCi() == false); + filter.setFailOnNoMatchingTests(buildParams.getCi() == false); }); }); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java index a934164d11af6..08484346e6908 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java @@ -33,7 +33,7 @@ public void apply(Project project) { var buildParams = loadBuildParams(project).get(); NamedDomainObjectContainer fixtures = project.container(TestFixtureDeployment.class); project.getExtensions().add("dockerFixtures", fixtures); - registerDeployTaskPerFixture(project, fixtures, buildParams.isCi()); + registerDeployTaskPerFixture(project, fixtures, buildParams.getCi()); project.getTasks().register(DEPLOY_FIXTURE_TASK_NAME, task -> task.dependsOn(project.getTasks().withType(DockerBuildTask.class))); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java index ab28a66d93065..2c40b56c00b9a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java @@ -131,7 +131,7 @@ public void apply(Project project) { tasks.withType(ComposeUp.class).named("composeUp").configure(t -> { // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions - if (buildParams.isCi()) { + if (buildParams.getCi()) { t.usesService(dockerComposeThrottle); t.usesService(dockerSupport); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java index b1f806b384374..37bb96716b3f6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java @@ -29,7 +29,7 @@ */ public abstract class ArchivedOracleJdkToolchainResolver extends AbstractCustomJavaToolchainResolver { - private static final Map ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1"); + private static final Map ARCHIVED_BASE_VERSIONS = Maps.of(21, "21.0.6", 20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1"); @Override public Optional resolve(JavaToolchainRequest request) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java index bb26bfd16721d..41a47ece90d5d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java @@ -33,11 +33,15 @@ interface JdkBuild { String url(String os, String arch, String extension); } - record ReleasedJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) implements JdkBuild { + record ReleaseJdkBuild(JavaLanguageVersion languageVersion, String host, String version, String buildNumber, String hash) + implements + JdkBuild { @Override public String url(String os, String arch, String extension) { - return "https://download.oracle.com/java/GA/jdk" + return "https://" + + host + + "/java/GA/jdk" + version + "/" + hash @@ -54,10 +58,9 @@ public String url(String os, String arch, String extension) { } } - record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion) implements JdkBuild { + record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String buildNumber) implements JdkBuild { @Override public String url(String os, String arch, String extension) { - String buildNumber = resolveBuildNumber(languageVersion.asInt()); return "https://download.java.net/java/early_access/jdk" + languageVersion.asInt() + "/" @@ -73,29 +76,6 @@ public String url(String os, String arch, String extension) { + "_bin." + extension; } - - private static String resolveBuildNumber(int version) { - String buildNumber = System.getProperty("runtime.java." + version + ".build"); - if (buildNumber != null) { - System.out.println("buildNumber = " + buildNumber); - return buildNumber; - } - buildNumber = System.getProperty("runtime.java.build"); - if (buildNumber != null) { - System.out.println("buildNumber2 = " + buildNumber); - return buildNumber; - } - - switch (version) { - case 24: - // latest explicitly found build number for 24 - return "29"; - case 25: - return "3"; - default: - throw new IllegalArgumentException("Unsupported version " + version); - } - } } private static final Pattern VERSION_PATTERN = Pattern.compile( @@ -110,14 +90,20 @@ private static String resolveBuildNumber(int version) { // package private so it can be replaced by tests List builds = List.of( - getBundledJdkBuild(), - new EarlyAccessJdkBuild(JavaLanguageVersion.of(24)), - new EarlyAccessJdkBuild(JavaLanguageVersion.of(25)) + getBundledJdkBuild(VersionProperties.getBundledJdkVersion(), VersionProperties.getBundledJdkMajorVersion()), + getEarlyAccessBuild(JavaLanguageVersion.of(25), "3") ); - private JdkBuild getBundledJdkBuild() { - String bundledJdkVersion = VersionProperties.getBundledJdkVersion(); - JavaLanguageVersion bundledJdkMajorVersion = JavaLanguageVersion.of(VersionProperties.getBundledJdkMajorVersion()); + static EarlyAccessJdkBuild getEarlyAccessBuild(JavaLanguageVersion languageVersion, String buildNumber) { + // first try the unversioned override, then the versioned override which has higher precedence + buildNumber = System.getProperty("runtime.java.build", buildNumber); + buildNumber = System.getProperty("runtime.java." + languageVersion.asInt() + ".build", buildNumber); + + return new EarlyAccessJdkBuild(languageVersion, buildNumber); + } + + static JdkBuild getBundledJdkBuild(String bundledJdkVersion, String bundledJkdMajorVersionString) { + JavaLanguageVersion bundledJdkMajorVersion = JavaLanguageVersion.of(bundledJkdMajorVersionString); Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(bundledJdkVersion); if (jdkVersionMatcher.matches() == false) { throw new IllegalStateException("Unable to parse bundled JDK version " + bundledJdkVersion); @@ -125,7 +111,7 @@ private JdkBuild getBundledJdkBuild() { String baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); String build = jdkVersionMatcher.group(3); String hash = jdkVersionMatcher.group(5); - return new ReleasedJdkBuild(bundledJdkMajorVersion, baseVersion, build, hash); + return new ReleaseJdkBuild(bundledJdkMajorVersion, "download.oracle.com", baseVersion, build, hash); } /** diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 9c57ca327c7b7..90c5b336175a0 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.12 \ No newline at end of file +8.13 \ No newline at end of file diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy index 4993bf00f2af5..9de9cea65a393 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy @@ -30,13 +30,17 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { return null } } - toolChain.builds = toolChain.builds.findAll { it instanceof OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild } + [ - new OracleOpenJdkToolchainResolver.ReleasedJdkBuild( + toolChain.builds = [ + new OracleOpenJdkToolchainResolver.ReleaseJdkBuild( JavaLanguageVersion.of(20), + "download.oracle.com", "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d" - )] + ), + OracleOpenJdkToolchainResolver.getBundledJdkBuild("24+36@1f9ff9062db4449d8ca828c504ffae90", "24"), + OracleOpenJdkToolchainResolver.getEarlyAccessBuild(JavaLanguageVersion.of(25), "3") + ] toolChain } @@ -51,17 +55,28 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { [20, anyVendor(), LINUX, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-x64_bin.tar.gz"], [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"], [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"], - // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz - [24, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"], - [24, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"], - [24, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"], - [24, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"], - [24, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"], - [24, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"], - [24, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"], - [24, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"], - [24, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"], - [24, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"]] + // bundled jdk + [24, ORACLE, MAC_OS, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-x64_bin.tar.gz"], + [24, ORACLE, MAC_OS, AARCH64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-aarch64_bin.tar.gz"], + [24, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz"], + [24, ORACLE, LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-aarch64_bin.tar.gz"], + [24, ORACLE, WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip"], + [24, anyVendor(), MAC_OS, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-x64_bin.tar.gz"], + [24, anyVendor(), MAC_OS, AARCH64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-aarch64_bin.tar.gz"], + [24, anyVendor(), LINUX, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz"], + [24, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-aarch64_bin.tar.gz"], + [24, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip"], + // EA build + [25, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-x64_bin.tar.gz"], + [25, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-aarch64_bin.tar.gz"], + [25, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-x64_bin.tar.gz"], + [25, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-aarch64_bin.tar.gz"], + [25, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_windows-x64_bin.zip"], + [25, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-x64_bin.tar.gz"], + [25, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_macos-aarch64_bin.tar.gz"], + [25, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-x64_bin.tar.gz"], + [25, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_linux-aarch64_bin.tar.gz"], + [25, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk25/3/GPL/openjdk-25-ea+3_windows-x64_bin.zip"]] } @RestoreSystemProperties @@ -85,16 +100,6 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { where: version | vendor | os | arch | expectedUrl - 24 | ORACLE | MAC_OS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz" - 24 | ORACLE | MAC_OS | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz" - 24 | ORACLE | LINUX | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz" - 24 | ORACLE | LINUX | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz" - 24 | ORACLE | WINDOWS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip" - 24 | anyVendor() | MAC_OS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz" - 24 | anyVendor() | MAC_OS | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz" - 24 | anyVendor() | LINUX | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz" - 24 | anyVendor() | LINUX | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz" - 24 | anyVendor() | WINDOWS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip" 25 | ORACLE | MAC_OS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz" 25 | ORACLE | MAC_OS | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz" 25 | ORACLE | LINUX | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz" diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java index 9e4f1cd3a913d..d5060a2e62365 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java @@ -239,6 +239,96 @@ public void updateVersionFile_removesCorrectly() throws Exception { assertThat(field.isPresent(), is(false)); } + @Test + public void addTransportVersion() throws Exception { + var transportVersions = """ + public class TransportVersions { + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var expectedTransportVersions = """ + public class TransportVersions { + + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + + public static final TransportVersion NEXT_TRANSPORT_VERSION = def(1_005_0_00); + + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var unit = StaticJavaParser.parse(transportVersions); + var result = UpdateVersionsTask.addTransportVersionConstant(unit, "NEXT_TRANSPORT_VERSION", 1_005_0_00); + + assertThat(result.isPresent(), is(true)); + assertThat(result.get(), hasToString(expectedTransportVersions)); + } + + @Test + public void addTransportVersionPatch() throws Exception { + var transportVersions = """ + public class TransportVersions { + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var expectedTransportVersions = """ + public class TransportVersions { + + public static final TransportVersion V_1_0_0 = def(1_000_0_00); + + public static final TransportVersion V_1_1_0 = def(1_001_0_00); + + public static final TransportVersion V_1_2_0 = def(1_002_0_00); + + public static final TransportVersion V_1_2_1 = def(1_002_0_01); + + public static final TransportVersion V_1_2_2 = def(1_002_0_02); + + public static final TransportVersion SOME_OTHER_VERSION = def(1_003_0_00); + + public static final TransportVersion PATCH_TRANSPORT_VERSION = def(1_003_0_01); + + public static final TransportVersion YET_ANOTHER_VERSION = def(1_004_0_00); + + public static final TransportVersion MINIMUM_COMPATIBLE = V_1_0_0; + } + """; + + var unit = StaticJavaParser.parse(transportVersions); + var result = UpdateVersionsTask.addTransportVersionConstant(unit, "PATCH_TRANSPORT_VERSION", 1_003_0_01); + + assertThat(result.isPresent(), is(true)); + assertThat(result.get(), hasToString(expectedTransportVersions)); + } + private static Optional findFirstField(Node node, String name) { return node.findFirst(FieldDeclaration.class, f -> f.getVariable(0).getName().getIdentifier().equals(name)); } diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 9d404cc4127b3..3f0f80cb92ba1 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -2,7 +2,7 @@ elasticsearch = 8.18.0 lucene = 9.12.1 bundled_jdk_vendor = openjdk -bundled_jdk = 23+37@3c5b90190c68498b986a97f276efd28a +bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90 # optional dependencies spatial4j = 0.7 jts = 1.15.0 @@ -14,7 +14,7 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 -netty = 4.1.115.Final +netty = 4.1.118.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 awsv1sdk = 1.12.270 @@ -29,7 +29,7 @@ opensaml = 4.3.0 # client dependencies httpclient = 4.5.14 -httpcore = 4.4.13 +httpcore = 4.4.16 httpasyncclient = 4.1.5 commonslogging = 1.2 commonscodec = 1.15 diff --git a/build-tools/settings.gradle b/build-tools/settings.gradle index fce1f3b33d524..87d61b40a7cca 100644 --- a/build-tools/settings.gradle +++ b/build-tools/settings.gradle @@ -9,7 +9,9 @@ pluginManagement { includeBuild "../build-conventions" } - +plugins { + id "com.gradle.develocity" version "3.19.2" +} include 'reaper' dependencyResolutionManagement { diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java index 42e576012c0c9..a1c003c4c315d 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java @@ -56,6 +56,7 @@ public class BasePluginBuildPlugin implements Plugin { public static final String BUNDLE_PLUGIN_TASK_NAME = "bundlePlugin"; public static final String EXPLODED_BUNDLE_PLUGIN_TASK_NAME = "explodedBundlePlugin"; public static final String EXPLODED_BUNDLE_CONFIG = "explodedBundleZip"; + public static final Attribute EXPLODED_PLUGIN_BUNDLE_ATTRIBUTE = Attribute.of("exploded-plugin-bundle", Boolean.class); protected final ProviderFactory providerFactory; @@ -170,6 +171,7 @@ private TaskProvider createBundleTasks(final Project project, PluginPropert explodedBundleZip.setCanBeResolved(false); explodedBundleZip.setCanBeConsumed(true); explodedBundleZip.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); + explodedBundleZip.getAttributes().attribute(EXPLODED_PLUGIN_BUNDLE_ATTRIBUTE, true); project.getArtifacts().add(EXPLODED_BUNDLE_CONFIG, explodedBundle); return bundle; } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 4cb67e249b0b0..705fda530256c 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -61,11 +61,14 @@ import java.io.UncheckedIOException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -1297,40 +1300,47 @@ private void syncWithCopy(Path sourceRoot, Path destinationRoot) { private void sync(Path sourceRoot, Path destinationRoot, BiConsumer syncMethod) { assert Files.exists(destinationRoot) == false; - try (Stream stream = Files.walk(sourceRoot)) { - stream.forEach(source -> { - Path relativeDestination = sourceRoot.relativize(source); - if (relativeDestination.getNameCount() <= 1) { - return; - } - // Throw away the first name as the archives have everything in a single top level folder we are not interested in - relativeDestination = relativeDestination.subpath(1, relativeDestination.getNameCount()); - - Path destination = destinationRoot.resolve(relativeDestination); - if (Files.isDirectory(source)) { - try { - Files.createDirectories(destination); - } catch (IOException e) { - throw new UncheckedIOException("Can't create directory " + destination.getParent(), e); + try { + Files.walkFileTree(sourceRoot, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path relativeDestination = sourceRoot.relativize(dir); + if (relativeDestination.getNameCount() <= 1) { + return FileVisitResult.CONTINUE; } - } else { - try { - Files.createDirectories(destination.getParent()); - } catch (IOException e) { - throw new UncheckedIOException("Can't create directory " + destination.getParent(), e); + // Throw away the first name as the archives have everything in a single top level folder we are not interested in + relativeDestination = relativeDestination.subpath(1, relativeDestination.getNameCount()); + Path destination = destinationRoot.resolve(relativeDestination); + Files.createDirectories(destination); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path source, BasicFileAttributes attrs) throws IOException { + Path relativeDestination = sourceRoot.relativize(source); + if (relativeDestination.getNameCount() <= 1) { + return FileVisitResult.CONTINUE; } + // Throw away the first name as the archives have everything in a single top level folder we are not interested in + relativeDestination = relativeDestination.subpath(1, relativeDestination.getNameCount()); + Path destination = destinationRoot.resolve(relativeDestination); + Files.createDirectories(destination.getParent()); syncMethod.accept(destination, source); + return FileVisitResult.CONTINUE; } - }); - } catch (UncheckedIOException e) { - if (e.getCause() instanceof NoSuchFileException cause) { - // Ignore these files that are sometimes left behind by the JVM - if (cause.getFile() == null || cause.getFile().contains(".attach_pid") == false) { - throw new UncheckedIOException(cause); + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + if (exc instanceof NoSuchFileException noFileException) { + // Ignore these files that are sometimes left behind by the JVM + if (noFileException.getFile() != null && noFileException.getFile().contains(".attach_pid")) { + LOGGER.info("Ignoring file left behind by JVM: {}", noFileException.getFile()); + return FileVisitResult.CONTINUE; + } + } + throw exc; } - } else { - throw e; - } + }); } catch (IOException e) { throw new UncheckedIOException("Can't walk source " + sourceRoot, e); } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java index 54bffd2a14b3d..07f88151a5154 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java @@ -24,6 +24,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -42,9 +43,11 @@ public abstract class RunTask extends DefaultTestClustersTask { private Boolean debug = false; private Boolean cliDebug = false; - private Boolean entitlementsEnabled = false; + private Boolean apmServerEnabled = false; + private List plugins = List.of(); + private Boolean preserveData = false; private Path dataDir = null; @@ -74,9 +77,7 @@ public void setCliDebug(boolean enabled) { option = "entitlements", description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies." ) - public void setEntitlementsEnabled(boolean enabled) { - this.entitlementsEnabled = enabled; - } + public void setEntitlementsEnabled(boolean enabled) {} @Input public Boolean getDebug() { @@ -90,7 +91,7 @@ public Boolean getCliDebug() { @Input public Boolean getEntitlementsEnabled() { - return entitlementsEnabled; + return true; } @Input @@ -103,6 +104,22 @@ public void setApmServerEnabled(Boolean apmServerEnabled) { this.apmServerEnabled = apmServerEnabled; } + @Option(option = "with-plugins", description = "Run distribution with plugins installed") + public void setPlugins(String plugins) { + this.plugins = Arrays.asList(plugins.split(",")); + for (var cluster : getClusters()) { + for (String plugin : this.plugins) { + cluster.plugin(":plugins:" + plugin); + } + dependsOn(cluster.getPluginAndModuleConfigurations()); + } + } + + @Input + public List getPlugins() { + return plugins; + } + @Option(option = "data-dir", description = "Override the base data directory used by the testcluster") public void setDataDir(String dataDirStr) { dataDir = Paths.get(dataDirStr).toAbsolutePath(); @@ -240,9 +257,7 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) { if (cliDebug) { enableCliDebug(); } - if (entitlementsEnabled) { - enableEntitlements(); - } + enableEntitlements(); } @TaskAction diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/transform/FilteringJarTransform.java b/build-tools/src/main/java/org/elasticsearch/gradle/transform/FilteringJarTransform.java new file mode 100644 index 0000000000000..1675dc3935f36 --- /dev/null +++ b/build-tools/src/main/java/org/elasticsearch/gradle/transform/FilteringJarTransform.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.transform; + +import org.gradle.api.Action; +import org.gradle.api.artifacts.dsl.DependencyHandler; +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Input; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.io.UncheckedIOException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +public abstract class FilteringJarTransform implements TransformAction { + public static final String FILTERED_JAR_TYPE = "filtered-jar"; + + @InputArtifact + public abstract Provider getInputArtifact(); + + @Override + public void transform(TransformOutputs outputs) { + File original = getInputArtifact().get().getAsFile(); + File transformed = outputs.file(original.getName()); + List excludes = createMatchers(getParameters().getExcludes()); + + try ( + ZipFile input = new ZipFile(original); + ZipOutputStream output = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(transformed))) + ) { + Enumeration entries = input.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (excludes.stream().noneMatch(e -> e.matches(Path.of(entry.getName())))) { + output.putNextEntry(entry); + input.getInputStream(entry).transferTo(output); + output.closeEntry(); + } + } + + output.flush(); + output.finish(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to patch archive", e); + } + } + + private List createMatchers(List patterns) { + return patterns.stream().map(p -> FileSystems.getDefault().getPathMatcher("glob:" + p)).toList(); + } + + public static void registerTransform(DependencyHandler dependencyHandler, Action config) { + dependencyHandler.registerTransform(FilteringJarTransform.class, spec -> { + spec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JAR_TYPE); + spec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, FILTERED_JAR_TYPE); + config.execute(spec.getParameters()); + }); + } + + public abstract static class Parameters implements TransformParameters, Serializable { + private List excludes = new ArrayList<>(); + + @Input + public List getExcludes() { + return excludes; + } + + public void exclude(String exclude) { + excludes.add(exclude); + } + } +} diff --git a/build.gradle b/build.gradle index 9d22d196a3ca4..09cc9529da5a6 100644 --- a/build.gradle +++ b/build.gradle @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin import org.elasticsearch.gradle.internal.ResolveAllDependencies import org.elasticsearch.gradle.util.GradleUtils @@ -118,10 +119,10 @@ tasks.register("updateCIBwcVersions") { outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipeline } - // Writes a Buildkite pipelime from a template, and replaces $BWC_LIST with an array of versions + // Writes a Buildkite pipelime from a template, and replaces a variable with an array of versions // Useful for writing a list of versions in a matrix configuration - def expandBwcList = { String outputFilePath, String pipelineTemplatePath, List versions -> - writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: "BWC_LIST")]) + def expandList = { String outputFilePath, String pipelineTemplatePath, String variable, List versions -> + writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: variable)]) } // Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version @@ -133,11 +134,18 @@ tasks.register("updateCIBwcVersions") { doLast { writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible)) writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible)) - expandBwcList( + expandList( ".buildkite/pipelines/intake.yml", ".buildkite/pipelines/intake.template.yml", + "BWC_LIST", filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible) ) + expandList( + ".buildkite/pipelines/periodic-fwc.yml", + ".buildkite/pipelines/periodic-fwc.template.yml", + "FWC_LIST", + buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor } + ) writeBuildkitePipeline( ".buildkite/pipelines/periodic.yml", ".buildkite/pipelines/periodic.template.yml", @@ -210,6 +218,22 @@ tasks.register("verifyVersions") { } } +def generateUpgradeCompatibilityFile = tasks.register("generateUpgradeCompatibilityFile") { + def outputFile = project.layout.buildDirectory.file("rolling-upgrade-compatible-${VersionProperties.elasticsearch}.json") + def rollingUpgradeCompatibleVersions = buildParams.bwcVersions.wireCompatible - VersionProperties.elasticsearchVersion + inputs.property("rollingUpgradeCompatibleVersions", rollingUpgradeCompatibleVersions) + outputs.file(outputFile) + doLast { + def versionsString = rollingUpgradeCompatibleVersions.collect { "\"${it.toString()}\"" }.join(', ') + outputFile.get().asFile.write("""{"rolling_upgrade_compatible_versions" : [${versionsString}]}""") + } +} + +def upgradeCompatibilityZip = tasks.register("upgradeCompatibilityZip", Zip) { + archiveFile.set(project.layout.buildDirectory.file("distributions/rolling-upgrade-compatible-${VersionProperties.elasticsearch}.zip")) + from(generateUpgradeCompatibilityFile) +} + /* * When adding backcompat behavior that spans major versions, temporarily * disabling the backcompat tests is necessary. This flag controls @@ -250,15 +274,6 @@ allprojects { } } - // injecting groovy property variables into all projects - project.ext { - // for ide hacks... - isEclipse = providers.systemProperty("eclipse.launcher").isPresent() || // Detects gradle launched from Eclipse's IDE - providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server - gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff - gradle.startParameter.taskNames.contains('cleanEclipse') - } - ext.bwc_tests_enabled = bwc_tests_enabled // eclipse configuration @@ -392,6 +407,34 @@ allprojects { apply plugin: 'elasticsearch.formatting' } +tasks.named("updateDaemonJvm") { + def myPlatforms = [ + BuildPlatformFactory.of( + org.gradle.platform.Architecture.AARCH64, + org.gradle.platform.OperatingSystem.MAC_OS + ), + BuildPlatformFactory.of( + org.gradle.platform.Architecture.AARCH64, + org.gradle.platform.OperatingSystem.LINUX + ), + BuildPlatformFactory.of( + org.gradle.platform.Architecture.X86_64, + org.gradle.platform.OperatingSystem.LINUX + ), + BuildPlatformFactory.of( + org.gradle.platform.Architecture.X86_64, + org.gradle.platform.OperatingSystem.WINDOWS + ), + // anyone still using x86 osx? + BuildPlatformFactory.of( + org.gradle.platform.Architecture.X86_64, + org.gradle.platform.OperatingSystem.MAC_OS + ) + ] + toolchainPlatforms.set(myPlatforms) + languageVersion = JavaLanguageVersion.of(21) + vendor = JvmVendorSpec.ADOPTIUM +} tasks.register("verifyBwcTestsEnabled") { doLast { @@ -487,6 +530,7 @@ tasks.register("buildReleaseArtifacts").configure { } .collect { GradleUtils.findByName(it.tasks, 'assemble') } .findAll { it != null } + dependsOn upgradeCompatibilityZip } tasks.register("spotlessApply").configure { diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 3fb2aa6595869..7a07f50e4b98d 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -37,7 +37,7 @@ base { // LLRC is licenses under Apache 2.0 projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) -licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) +licenseFile.set(layout.getSettingsDirectory().file('licenses/APACHE-LICENSE-2.0.txt').asFile) dependencies { api "org.apache.httpcomponents:httpclient:${versions.httpclient}" diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 9b1cb1140311b..38fc949a5c7c8 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -33,7 +33,7 @@ base { // rest client sniffer is licenses under Apache 2.0 projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) -licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) +licenseFile.set(layout.getSettingsDirectory().file('licenses/APACHE-LICENSE-2.0.txt').asFile) dependencies { api project(":client:rest") diff --git a/client/test/build.gradle b/client/test/build.gradle index e39b7587b69d5..27b1577ce3098 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -19,7 +19,7 @@ group = "${group}.client.test" // rest client sniffer is licenses under Apache 2.0 projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) -licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) +licenseFile.set(layout.getSettingsDirectory().file('licenses/APACHE-LICENSE-2.0.txt').asFile) dependencies { api "org.apache.httpcomponents:httpcore:${versions.httpcore}" @@ -28,9 +28,9 @@ dependencies { api "org.hamcrest:hamcrest:${versions.hamcrest}" // mockito - api 'org.mockito:mockito-core:5.11.0' - api 'org.mockito:mockito-subclass:5.11.0' - api 'net.bytebuddy:byte-buddy:1.14.12' + api 'org.mockito:mockito-core:5.15.2' + api 'org.mockito:mockito-subclass:5.15.2' + api 'net.bytebuddy:byte-buddy:1.15.11' api 'org.objenesis:objenesis:3.3' } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 7bbfb0f313e55..ddfdaa69bb392 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -57,13 +57,13 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b pluginsDir.getParent() } } - from(rootProject.projectDir) { + from(layout.settingsDirectory.asFile) { filePermissions { unix(0644) } include 'README.asciidoc' } - from(rootProject.file('licenses')) { + from(layout.settingsDirectory.file('licenses').asFile) { include isTestDistro ? 'AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt' : 'ELASTIC-LICENSE-2.0.txt' filePermissions { unix(0644) diff --git a/distribution/build.gradle b/distribution/build.gradle index 5fc2846b031fb..10a19f3a98862 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -15,6 +15,7 @@ import org.elasticsearch.gradle.internal.ConcatFilesTask import org.elasticsearch.gradle.internal.DependenciesInfoPlugin import org.elasticsearch.gradle.internal.NoticeTask import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin +import org.elasticsearch.gradle.transform.FilteringJarTransform import java.nio.file.Files import java.nio.file.Path @@ -261,7 +262,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * Properties to expand when copying packaging files * *****************************************************************************/ configurations { - ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent', 'libsEntitlementBridge'].each { + ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent'].each { create(it) { canBeConsumed = false canBeResolved = true @@ -272,12 +273,28 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } } + libsEntitlementBridge { + canBeConsumed = false + canBeResolved = true + attributes { + attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) + attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) + attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) + attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, FilteringJarTransform.FILTERED_JAR_TYPE) + } + } all { resolutionStrategy.dependencySubstitution { - substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:log4j") because "patched to remove JndiLookup clas"} + substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:log4j") because "patched to remove JndiLookup class"} } } + // Register artifact transform for filtering entitlements-bridge jar + FilteringJarTransform.registerTransform(dependencies) { spec -> + spec.exclude('module-info.class') + spec.exclude('META-INF/versions/**') + } + dependencies { libs project(':server') @@ -377,7 +394,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude "**/platform/${excludePlatform}/**" } } - if (buildParams.isSnapshotBuild()) { + if (buildParams.getSnapshotBuild()) { from(buildExternalTestModulesTaskProvider) } if (project.path.startsWith(':distribution:packages')) { @@ -518,9 +535,9 @@ subprojects { String licenseText if (isTestDistro) { - licenseText = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').getText('UTF-8') + licenseText = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile.getText('UTF-8') } else { - licenseText = rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt').getText('UTF-8') + licenseText = layout.settingsDirectory.file('licenses/ELASTIC-LICENSE-2.0.txt').asFile.getText('UTF-8') } // license text needs to be indented with a single space licenseText = ' ' + licenseText.replace('\n', '\n ') diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 36aaf4e5625b7..f62ca3ad70d73 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -119,7 +119,7 @@ ext.expansions = { Architecture architecture, DockerBase base -> // the image. When developing the Docker images, it's very tedious to completely rebuild // an image for every single change. Therefore, outside of CI, we fix the // build time to midnight so that the Docker build cache is usable. - def buildDate = buildParams.isCi() ? buildParams.buildDate : buildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString() + def buildDate = buildParams.ci ? buildParams.buildDate : buildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString() return [ 'arch' : architecture.classifier, @@ -389,7 +389,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() }) - noCache = buildParams.isCi() + noCache = buildParams.ci tags = generateTags(base, architecture) platforms.add(architecture.dockerPlatform) @@ -399,8 +399,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { // is functional. if (base == DockerBase.IRON_BANK) { Map buildArgsMap = [ - 'BASE_REGISTRY': 'docker.elastic.co', - 'BASE_IMAGE' : 'ubi9/ubi', + 'BASE_REGISTRY': 'docker.io', + 'BASE_IMAGE' : 'redhat/ubi9', 'BASE_TAG' : 'latest' ] @@ -484,7 +484,7 @@ void addBuildEssDockerImageTask(Architecture architecture) { dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() }) - noCache = buildParams.isCi() + noCache = buildParams.ci baseImages = [] tags = generateTags(dockerBase, architecture) platforms.add(architecture.dockerPlatform) diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml index 19b4a13dc9f22..0a2d3ff921675 100644 --- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml +++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml @@ -47,7 +47,7 @@ maintainers: - name: "Mark Vieira" email: "mark.vieira@elastic.co" username: "mark-vieira" - - name: "Rene Gröschke" + - name: "Rene Groeschke" email: "rene.groschke@elastic.co" username: "breskeby" - email: "klepal_alexander@bah.com" diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 9cf718b0c992a..3cd954c6473c9 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -44,7 +44,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.10.1" + alias(buildLibs.plugins.ospackage) } ['deb', 'rpm'].each { type -> @@ -175,7 +175,7 @@ def commonPackageConfig(String type, String architecture) { } else { assert type == 'rpm' into('/usr/share/elasticsearch') { - from(rootProject.file('licenses')) { + from(layout.settingsDirectory.file('licenses').asFile) { include 'ELASTIC-LICENSE-2.0.txt' rename { 'LICENSE.txt' } } @@ -301,7 +301,7 @@ ospackage { url = 'https://www.elastic.co/' // signing setup - if (project.hasProperty('signing.password') && buildParams.isSnapshotBuild() == false) { + if (project.hasProperty('signing.password') && buildParams.snapshotBuild == false) { signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4' signingKeyPassphrase = project.property('signing.password') signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ? diff --git a/distribution/src/bin/elasticsearch-env b/distribution/src/bin/elasticsearch-env index a89495fc5d144..4982f8dc3ac54 100644 --- a/distribution/src/bin/elasticsearch-env +++ b/distribution/src/bin/elasticsearch-env @@ -55,11 +55,17 @@ else JAVA_TYPE="bundled JDK" fi -# do not let JAVA_TOOL_OPTIONS slip in (as the JVM does by default) +# do not let JAVA_TOOL_OPTIONS OR _JAVA_OPTIONS slip in (as the JVM does by default) if [ ! -z "$JAVA_TOOL_OPTIONS" ]; then - echo "warning: ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS" + echo -n "warning: ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS; " + echo "pass JVM parameters via ES_JAVA_OPTS" unset JAVA_TOOL_OPTIONS fi +if [ ! -z "$_JAVA_OPTIONS" ]; then + echo -n "warning: ignoring _JAVA_OPTIONS=$_JAVA_OPTIONS; " + echo "pass JVM parameters via ES_JAVA_OPTS" + unset _JAVA_OPTIONS +fi # warn that we are not observing the value of JAVA_HOME if [ ! -z "$JAVA_HOME" ]; then diff --git a/distribution/src/bin/elasticsearch-env.bat b/distribution/src/bin/elasticsearch-env.bat index 93fc26802a53c..ff7db68d1f54a 100644 --- a/distribution/src/bin/elasticsearch-env.bat +++ b/distribution/src/bin/elasticsearch-env.bat @@ -58,11 +58,17 @@ if defined ES_JAVA_HOME ( set JAVA_TYPE=bundled JDK ) -rem do not let JAVA_TOOL_OPTIONS slip in (as the JVM does by default) +rem do not let JAVA_TOOL_OPTIONS or _JAVA_OPTIONS slip in (as the JVM does by default) if defined JAVA_TOOL_OPTIONS ( - echo warning: ignoring JAVA_TOOL_OPTIONS=%JAVA_TOOL_OPTIONS% + (echo|set /p=ignoring JAVA_TOOL_OPTIONS=%JAVA_TOOL_OPTIONS%; ) + echo pass JVM parameters via ES_JAVA_OPTS set JAVA_TOOL_OPTIONS= ) +if defined _JAVA_OPTIONS ( + (echo|set /p=ignoring _JAVA_OPTIONS=%_JAVA_OPTIONS%; ) + echo pass JVM parameters via ES_JAVA_OPTS + set _JAVA_OPTIONS= +) rem warn that we are not observing the value of $JAVA_HOME if defined JAVA_HOME ( diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java index cc662bd747575..c6421d76392cf 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java @@ -74,7 +74,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment keyStore.setFile(setting, Files.readAllBytes(file)); } - keyStore.save(env.configFile(), getKeyStorePassword().getChars()); + keyStore.save(env.configDir(), getKeyStorePassword().getChars()); } @SuppressForbidden(reason = "file arg for cli") diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java index c01c18418858a..a7ea6dcf7ce74 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java @@ -100,7 +100,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment } } - keyStore.save(env.configFile(), getKeyStorePassword().getChars()); + keyStore.save(env.configDir(), getKeyStorePassword().getChars()); } } diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java index 0380018d36cff..a8a75ac23c900 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java @@ -39,14 +39,14 @@ public BaseKeyStoreCommand(String description, boolean keyStoreMustExist) { @Override public final void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { try { - final Path configFile = env.configFile(); + final Path configFile = env.configDir(); keyStore = KeyStoreWrapper.load(configFile); if (keyStore == null) { if (keyStoreMustExist) { throw new UserException( ExitCodes.DATA_ERROR, "Elasticsearch keystore not found at [" - + KeyStoreWrapper.keystorePath(env.configFile()) + + KeyStoreWrapper.keystorePath(env.configDir()) + "]. Use 'create' command to create one." ); } else if (options.has(forceOption) == false) { diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java index 4dca3d538263a..9e4f70eee559d 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java @@ -31,7 +31,7 @@ class ChangeKeyStorePasswordCommand extends BaseKeyStoreCommand { protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception { try (SecureString newPassword = readPassword(terminal, true)) { final KeyStoreWrapper keyStore = getKeyStore(); - keyStore.save(env.configFile(), newPassword.getChars()); + keyStore.save(env.configDir(), newPassword.getChars()); terminal.println("Elasticsearch keystore password changed successfully."); } catch (SecurityException e) { throw new UserException(ExitCodes.DATA_ERROR, e.getMessage()); diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java index a922c92f5f44b..ef561b08d9a50 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java @@ -40,7 +40,7 @@ class CreateKeyStoreCommand extends KeyStoreAwareCommand { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { try (SecureString password = options.has(passwordOption) ? readPassword(terminal, true) : new SecureString(new char[0])) { - Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile()); + Path keystoreFile = KeyStoreWrapper.keystorePath(env.configDir()); if (Files.exists(keystoreFile)) { if (terminal.promptYesNo("An elasticsearch keystore already exists. Overwrite?", false) == false) { terminal.println("Exiting without creating keystore."); @@ -48,8 +48,8 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce } } KeyStoreWrapper keystore = KeyStoreWrapper.create(); - keystore.save(env.configFile(), password.getChars()); - terminal.println("Created elasticsearch keystore in " + KeyStoreWrapper.keystorePath(env.configFile())); + keystore.save(env.configDir(), password.getChars()); + terminal.println("Created elasticsearch keystore in " + KeyStoreWrapper.keystorePath(env.configDir())); } catch (SecurityException e) { throw new UserException(ExitCodes.IO_ERROR, "Error creating the elasticsearch keystore."); } diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java index 0428d5dcf7df8..f0eaca1648b96 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java @@ -32,7 +32,7 @@ public class HasPasswordKeyStoreCommand extends KeyStoreAwareCommand { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { - final Path configFile = env.configFile(); + final Path configFile = env.configDir(); final KeyStoreWrapper keyStore = KeyStoreWrapper.load(configFile); // We handle error printing here so we can respect the "--silent" flag diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java index 8a973c6d67f7d..fb1a2ad1df7f3 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java @@ -45,6 +45,6 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment } keyStore.remove(setting); } - keyStore.save(env.configFile(), getKeyStorePassword().getChars()); + keyStore.save(env.configDir(), getKeyStorePassword().getChars()); } } diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java index b7061d6153b80..bbbfbf81f7ed9 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java @@ -26,7 +26,7 @@ public class UpgradeKeyStoreCommand extends BaseKeyStoreCommand { @Override protected void executeCommand(final Terminal terminal, final OptionSet options, final Environment env) throws Exception { - KeyStoreWrapper.upgrade(getKeyStore(), env.configFile(), getKeyStorePassword().getChars()); + KeyStoreWrapper.upgrade(getKeyStore(), env.configDir(), getKeyStorePassword().getChars()); } } diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java index edd70e4e52f55..56706dd44f0c3 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java @@ -46,14 +46,14 @@ private Path createRandomFile() throws IOException { for (int i = 0; i < length; ++i) { bytes[i] = randomByte(); } - Path file = env.configFile().resolve(randomAlphaOfLength(16)); + Path file = env.configDir().resolve(randomAlphaOfLength(16)); Files.write(file, bytes); return file; } private void addFile(KeyStoreWrapper keystore, String setting, Path file, String password) throws Exception { keystore.setFile(setting, Files.readAllBytes(file)); - keystore.save(env.configFile(), password.toCharArray()); + keystore.save(env.configDir(), password.toCharArray()); } public void testMissingCreateWithEmptyPasswordWhenPrompted() throws Exception { @@ -77,7 +77,7 @@ public void testMissingNoCreate() throws Exception { terminal.addSecretInput(randomFrom("", "keystorepassword")); terminal.addTextInput("n"); // explicit no execute("foo"); - assertNull(KeyStoreWrapper.load(env.configFile())); + assertNull(KeyStoreWrapper.load(env.configDir())); } public void testOverwritePromptDefault() throws Exception { diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java index 3de18e094104f..412624be1d506 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java @@ -83,7 +83,7 @@ public void testMissingPromptCreateWithoutPasswordWithoutPromptIfForced() throws public void testMissingNoCreate() throws Exception { terminal.addTextInput("n"); // explicit no execute("foo"); - assertNull(KeyStoreWrapper.load(env.configFile())); + assertNull(KeyStoreWrapper.load(env.configDir())); } public void testOverwritePromptDefault() throws Exception { @@ -143,7 +143,7 @@ public void testForceNonExistent() throws Exception { public void testPromptForValue() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); terminal.addSecretInput("secret value"); execute("foo"); @@ -152,7 +152,7 @@ public void testPromptForValue() throws Exception { public void testPromptForMultipleValues() throws Exception { final String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); terminal.addSecretInput("bar1"); terminal.addSecretInput("bar2"); @@ -165,7 +165,7 @@ public void testPromptForMultipleValues() throws Exception { public void testStdinShort() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); setInput("secret value 1"); execute("-x", "foo"); @@ -174,7 +174,7 @@ public void testStdinShort() throws Exception { public void testStdinLong() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); setInput("secret value 2"); execute("--stdin", "foo"); @@ -183,7 +183,7 @@ public void testStdinLong() throws Exception { public void testStdinNoInput() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); setInput(""); execute("-x", "foo"); @@ -192,7 +192,7 @@ public void testStdinNoInput() throws Exception { public void testStdinInputWithLineBreaks() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); setInput("Typedthisandhitenter\n"); execute("-x", "foo"); @@ -201,7 +201,7 @@ public void testStdinInputWithLineBreaks() throws Exception { public void testStdinInputWithCarriageReturn() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); setInput("Typedthisandhitenter\r"); execute("-x", "foo"); @@ -210,7 +210,7 @@ public void testStdinInputWithCarriageReturn() throws Exception { public void testStdinWithMultipleValues() throws Exception { final String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); setInput("bar1\nbar2\nbar3"); execute(randomFrom("-x", "--stdin"), "foo1", "foo2", "foo3"); @@ -221,7 +221,7 @@ public void testStdinWithMultipleValues() throws Exception { public void testAddUtf8String() throws Exception { String password = "keystorepassword"; - KeyStoreWrapper.create().save(env.configFile(), password.toCharArray()); + KeyStoreWrapper.create().save(env.configDir(), password.toCharArray()); terminal.addSecretInput(password); final int stringSize = randomIntBetween(8, 16); try (CharArrayWriter secretChars = new CharArrayWriter(stringSize)) { diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java index 0fc76943f9d05..d93bc2466ed7b 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java @@ -42,7 +42,7 @@ public void setupEnv() throws IOException { public void testLoadSecureSettings() throws Exception { final char[] password = KeyStoreWrapperTests.getPossibleKeystorePassword(); - final Path configPath = env.configFile(); + final Path configPath = env.configDir(); final SecureString seed; try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { seed = KeyStoreWrapper.SEED_SETTING.get(Settings.builder().setSecureSettings(keyStoreWrapper).build()); diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java index 72a83a48b6344..74b8c634939fd 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java @@ -48,7 +48,7 @@ public void testNotMatchingPasswords() throws Exception { public void testDefaultNotPromptForPassword() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); execute(); - Path configDir = env.configFile(); + Path configDir = env.configDir(); assertNotNull(KeyStoreWrapper.load(configDir)); } @@ -63,7 +63,7 @@ public void testPosix() throws Exception { } else { execute(); } - Path configDir = env.configFile(); + Path configDir = env.configDir(); assertNotNull(KeyStoreWrapper.load(configDir)); } @@ -79,13 +79,13 @@ public void testNotPosix() throws Exception { } else { execute(); } - Path configDir = env.configFile(); + Path configDir = env.configDir(); assertNotNull(KeyStoreWrapper.load(configDir)); } public void testOverwrite() throws Exception { String password = getPossibleKeystorePassword(); - Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile()); + Path keystoreFile = KeyStoreWrapper.keystorePath(env.configDir()); byte[] content = "not a keystore".getBytes(StandardCharsets.UTF_8); Files.write(keystoreFile, content); @@ -110,6 +110,6 @@ public void testOverwrite() throws Exception { } else { execute(); } - assertNotNull(KeyStoreWrapper.load(env.configFile())); + assertNotNull(KeyStoreWrapper.load(env.configDir())); } } diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java index 80edce4a20796..fcbe7b2226296 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java @@ -77,11 +77,11 @@ KeyStoreWrapper createKeystore(String password, String... settings) throws Excep } void saveKeystore(KeyStoreWrapper keystore, String password) throws Exception { - keystore.save(env.configFile(), password.toCharArray()); + keystore.save(env.configDir(), password.toCharArray()); } KeyStoreWrapper loadKeystore(String password) throws Exception { - KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile()); + KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configDir()); keystore.decrypt(password.toCharArray()); return keystore; } diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java index 5ab27bac3998a..ee3a53d5c3df7 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java @@ -84,8 +84,8 @@ public void testFileSettingExhaustiveBytes() throws Exception { bytes[i] = (byte) i; } keystore.setFile("foo", bytes); - keystore.save(env.configFile(), password); - keystore = KeyStoreWrapper.load(env.configFile()); + keystore.save(env.configDir(), password); + keystore = KeyStoreWrapper.load(env.configDir()); keystore.decrypt(password); try (InputStream stream = keystore.getFile("foo")) { for (int i = 0; i < 256; ++i) { @@ -114,8 +114,8 @@ public void testDecryptKeyStoreWithWrongPassword() throws Exception { invalidPassword[realPassword.length] = '#'; } KeyStoreWrapper keystore = KeyStoreWrapper.create(); - keystore.save(env.configFile(), realPassword); - final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile()); + keystore.save(env.configDir(), realPassword); + final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configDir()); final SecurityException exception = expectThrows(SecurityException.class, () -> loadedkeystore.decrypt(invalidPassword)); if (inFipsJvm()) { assertThat( @@ -133,8 +133,8 @@ public void testDecryptKeyStoreWithWrongPassword() throws Exception { public void testDecryptKeyStoreWithShortPasswordInFips() throws Exception { assumeTrue("This should run only in FIPS mode", inFipsJvm()); KeyStoreWrapper keystore = KeyStoreWrapper.create(); - keystore.save(env.configFile(), "alongenoughpassword".toCharArray()); - final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile()); + keystore.save(env.configDir(), "alongenoughpassword".toCharArray()); + final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configDir()); final GeneralSecurityException exception = expectThrows( GeneralSecurityException.class, () -> loadedkeystore.decrypt("shortpwd".toCharArray()) // shorter than 14 characters @@ -147,7 +147,7 @@ public void testCreateKeyStoreWithShortPasswordInFips() throws Exception { KeyStoreWrapper keystore = KeyStoreWrapper.create(); final GeneralSecurityException exception = expectThrows( GeneralSecurityException.class, - () -> keystore.save(env.configFile(), "shortpwd".toCharArray()) // shorter than 14 characters + () -> keystore.save(env.configDir(), "shortpwd".toCharArray()) // shorter than 14 characters ); assertThat(exception.getMessage(), containsString("Error generating an encryption key from the provided password")); } @@ -192,18 +192,18 @@ public void testUpgradeNoop() throws Exception { final char[] password = getPossibleKeystorePassword(); KeyStoreWrapper keystore = KeyStoreWrapper.create(); SecureString seed = keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()); - keystore.save(env.configFile(), password); + keystore.save(env.configDir(), password); // upgrade does not overwrite seed - KeyStoreWrapper.upgrade(keystore, env.configFile(), password); + KeyStoreWrapper.upgrade(keystore, env.configDir(), password); assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString()); - keystore = KeyStoreWrapper.load(env.configFile()); + keystore = KeyStoreWrapper.load(env.configDir()); keystore.decrypt(password); assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString()); } public void testFailWhenCannotConsumeSecretStream() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); - Path configDir = env.configFile(); + Path configDir = env.configDir(); try ( Directory directory = newFSDirectory(configDir); IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT) @@ -234,7 +234,7 @@ public void testFailWhenCannotConsumeSecretStream() throws Exception { public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); - Path configDir = env.configFile(); + Path configDir = env.configDir(); try ( Directory directory = newFSDirectory(configDir); IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT) @@ -266,7 +266,7 @@ public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception { public void testFailWhenSecretStreamNotConsumed() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); - Path configDir = env.configFile(); + Path configDir = env.configDir(); try ( Directory directory = newFSDirectory(configDir); IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT) @@ -296,7 +296,7 @@ public void testFailWhenSecretStreamNotConsumed() throws Exception { public void testFailWhenEncryptedBytesStreamIsNotConsumed() throws Exception { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); - Path configDir = env.configFile(); + Path configDir = env.configDir(); try ( Directory directory = newFSDirectory(configDir); IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT) @@ -359,11 +359,11 @@ public void testUpgradeAddsSeed() throws Exception { final char[] password = getPossibleKeystorePassword(); KeyStoreWrapper keystore = KeyStoreWrapper.create(); keystore.remove(KeyStoreWrapper.SEED_SETTING.getKey()); - keystore.save(env.configFile(), password); - KeyStoreWrapper.upgrade(keystore, env.configFile(), password); + keystore.save(env.configDir(), password); + KeyStoreWrapper.upgrade(keystore, env.configDir(), password); SecureString seed = keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()); assertNotNull(seed); - keystore = KeyStoreWrapper.load(env.configFile()); + keystore = KeyStoreWrapper.load(env.configDir()); keystore.decrypt(password); assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString()); } @@ -380,7 +380,7 @@ public void testIllegalSettingName() throws Exception { public void testBackcompatV4() throws Exception { assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm()); - Path configDir = env.configFile(); + Path configDir = env.configDir(); try ( Directory directory = newFSDirectory(configDir); IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT) @@ -421,10 +421,10 @@ public void testStringAndFileDistinction() throws Exception { final Path temp = createTempDir(); Files.writeString(temp.resolve("file_setting"), "file_value", StandardCharsets.UTF_8); wrapper.setFile("file_setting", Files.readAllBytes(temp.resolve("file_setting"))); - wrapper.save(env.configFile(), password); + wrapper.save(env.configDir(), password); wrapper.close(); - final KeyStoreWrapper afterSave = KeyStoreWrapper.load(env.configFile()); + final KeyStoreWrapper afterSave = KeyStoreWrapper.load(env.configDir()); assertNotNull(afterSave); afterSave.decrypt(password); assertThat(afterSave.getSettingNames(), equalTo(Set.of("keystore.seed", "string_setting", "file_setting"))); @@ -510,8 +510,8 @@ public void testSerializationWhenLoadedFromFile() throws Exception { // testing with password and raw dataBytes[] final char[] password = getPossibleKeystorePassword(); - wrapper.save(env.configFile(), password); - final KeyStoreWrapper fromFile = KeyStoreWrapper.load(env.configFile()); + wrapper.save(env.configDir(), password); + final KeyStoreWrapper fromFile = KeyStoreWrapper.load(env.configDir()); fromFile.decrypt(password); assertThat(fromFile.getSettingNames(), hasSize(2)); diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java index bb533f32c7ac2..894b9d215a47f 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java @@ -62,11 +62,11 @@ private void assertKeystoreUpgradeWithPassword(String file, int version) throws } private void assertKeystoreUpgrade(String file, int version, @Nullable String password) throws Exception { - final Path keystore = KeyStoreWrapper.keystorePath(env.configFile()); + final Path keystore = KeyStoreWrapper.keystorePath(env.configDir()); try (InputStream is = KeyStoreWrapperTests.class.getResourceAsStream(file); OutputStream os = Files.newOutputStream(keystore)) { is.transferTo(os); } - try (KeyStoreWrapper beforeUpgrade = KeyStoreWrapper.load(env.configFile())) { + try (KeyStoreWrapper beforeUpgrade = KeyStoreWrapper.load(env.configDir())) { assertNotNull(beforeUpgrade); assertThat(beforeUpgrade.getFormatVersion(), equalTo(version)); } @@ -77,7 +77,7 @@ private void assertKeystoreUpgrade(String file, int version, @Nullable String pa execute(); terminal.reset(); - try (KeyStoreWrapper afterUpgrade = KeyStoreWrapper.load(env.configFile())) { + try (KeyStoreWrapper afterUpgrade = KeyStoreWrapper.load(env.configDir())) { assertNotNull(afterUpgrade); assertThat(afterUpgrade.getFormatVersion(), equalTo(KeyStoreWrapper.CURRENT_VERSION)); afterUpgrade.decrypt(password != null ? password.toCharArray() : new char[0]); @@ -87,6 +87,6 @@ private void assertKeystoreUpgrade(String file, int version, @Nullable String pa public void testKeystoreDoesNotExist() { final UserException e = expectThrows(UserException.class, this::execute); - assertThat(e, hasToString(containsString("keystore not found at [" + KeyStoreWrapper.keystorePath(env.configFile()) + "]"))); + assertThat(e, hasToString(containsString("keystore not found at [" + KeyStoreWrapper.keystorePath(env.configDir()) + "]"))); } } diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index dc2bcd96b8d9f..becdfbdb4d5e5 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -24,7 +24,8 @@ dependencies { compileOnly project(":libs:cli") implementation project(":libs:plugin-api") implementation project(":libs:plugin-scanner") - // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice + implementation project(":libs:entitlement") + // TODO: asm is picked up from the plugin scanner and entitlements, we should consolidate so it is not defined twice implementation 'org.ow2.asm:asm:9.7.1' implementation 'org.ow2.asm:asm-tree:9.7.1' diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java index d443cf5e1e181..0733fce0f5c77 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java @@ -24,8 +24,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; import org.elasticsearch.Build; -import org.elasticsearch.bootstrap.PluginPolicyInfo; -import org.elasticsearch.bootstrap.PolicyUtil; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; @@ -36,6 +34,7 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; +import org.elasticsearch.entitlement.runtime.policy.PolicyUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.jdk.JarHell; import org.elasticsearch.plugin.scanner.ClassReaders; @@ -249,8 +248,8 @@ public void execute(List plugins) throws Exception { final List deleteOnFailure = new ArrayList<>(); deleteOnFailures.put(pluginId, deleteOnFailure); - final Path pluginZip = download(plugin, env.tmpFile()); - final Path extractedZip = unzip(pluginZip, env.pluginsFile()); + final Path pluginZip = download(plugin, env.tmpDir()); + final Path extractedZip = unzip(pluginZip, env.pluginsDir()); deleteOnFailure.add(extractedZip); final PluginDescriptor pluginDescriptor = installPlugin(plugin, extractedZip, deleteOnFailure); terminal.println(logPrefix + "Installed " + pluginDescriptor.getName()); @@ -868,14 +867,14 @@ private PluginDescriptor loadPluginInfo(Path pluginRoot) throws Exception { PluginsUtils.verifyCompatibility(info); // checking for existing version of the plugin - verifyPluginName(env.pluginsFile(), info.getName()); + verifyPluginName(env.pluginsDir(), info.getName()); - PluginsUtils.checkForFailedPluginRemovals(env.pluginsFile()); + PluginsUtils.checkForFailedPluginRemovals(env.pluginsDir()); terminal.println(VERBOSE, info.toString()); // check for jar hell before any copying - jarHellCheck(info, pluginRoot, env.pluginsFile(), env.modulesFile()); + jarHellCheck(info, pluginRoot, env.pluginsDir(), env.modulesDir()); if (info.isStable() && hasNamedComponentFile(pluginRoot) == false) { generateNameComponentFile(pluginRoot); @@ -922,12 +921,22 @@ void jarHellCheck(PluginDescriptor candidateInfo, Path candidateDir, Path plugin */ private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoot, List deleteOnFailure) throws Exception { final PluginDescriptor info = loadPluginInfo(tmpRoot); - PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpFile()); - if (pluginPolicy != null) { - Set permissions = PluginSecurity.getPermissionDescriptions(pluginPolicy, env.tmpFile()); - PluginSecurity.confirmPolicyExceptions(terminal, permissions, batch); + + Path legacyPolicyFile = tmpRoot.resolve(PluginDescriptor.ES_PLUGIN_POLICY); + if (Files.exists(legacyPolicyFile)) { + terminal.errorPrintln( + "WARNING: this plugin contains a legacy Security Policy file. Starting with version 8.18, " + + "Entitlements replace SecurityManager as the security mechanism. Plugins must migrate their policy files to the new " + + "format. For more information, please refer to " + + PluginSecurity.ENTITLEMENTS_DESCRIPTION_URL + ); } + var pluginPolicy = PolicyUtils.parsePolicyIfExists(info.getName(), tmpRoot, true); + + Set entitlements = PolicyUtils.getEntitlementsDescriptions(pluginPolicy); + PluginSecurity.confirmPolicyExceptions(terminal, entitlements, batch); + // Validate that the downloaded plugin's ID matches what we expect from the descriptor. The // exception is if we install a plugin via `InstallPluginCommand` by specifying a URL or // Maven coordinates, because then we can't know in advance what the plugin ID ought to be. @@ -938,14 +947,14 @@ private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoo ); } - final Path destination = env.pluginsFile().resolve(info.getName()); + final Path destination = env.pluginsDir().resolve(info.getName()); deleteOnFailure.add(destination); installPluginSupportFiles( info, tmpRoot, - env.binFile().resolve(info.getName()), - env.configFile().resolve(info.getName()), + env.binDir().resolve(info.getName()), + env.configDir().resolve(info.getName()), deleteOnFailure ); movePlugin(tmpRoot, destination); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java index fc578c81b24c9..f51a478fe2135 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java @@ -40,13 +40,13 @@ class ListPluginsCommand extends EnvironmentAwareCommand { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { - if (Files.exists(env.pluginsFile()) == false) { - throw new IOException("Plugins directory missing: " + env.pluginsFile()); + if (Files.exists(env.pluginsDir()) == false) { + throw new IOException("Plugins directory missing: " + env.pluginsDir()); } - terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsFile()); + terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsDir()); final List plugins = new ArrayList<>(); - try (DirectoryStream paths = Files.newDirectoryStream(env.pluginsFile())) { + try (DirectoryStream paths = Files.newDirectoryStream(env.pluginsDir())) { for (Path path : paths) { if (path.getFileName().toString().equals(ELASTICSEARCH_PLUGINS_YML_CACHE) == false) { plugins.add(path); @@ -61,7 +61,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce private static void printPlugin(Environment env, Terminal terminal, Path plugin, String prefix) throws IOException { terminal.println(Terminal.Verbosity.SILENT, prefix + plugin.getFileName().toString()); - PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsFile().resolve(plugin)); + PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsDir().resolve(plugin)); terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix)); // When PluginDescriptor#getElasticsearchVersion returns a string, we can revisit the need diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java index 376c797d68899..47bc6145c61bf 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java @@ -9,65 +9,59 @@ package org.elasticsearch.plugins.cli; -import org.elasticsearch.bootstrap.PluginPolicyInfo; -import org.elasticsearch.bootstrap.PolicyUtil; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.Terminal.Verbosity; import org.elasticsearch.cli.UserException; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Path; -import java.security.Permission; -import java.security.UnresolvedPermission; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; /** - * Contains methods for displaying extended plugin permissions to the user, and confirming that + * Contains methods for displaying extended plugin entitlements to the user, and confirming that * plugin installation can proceed. */ public class PluginSecurity { + public static final String ENTITLEMENTS_DESCRIPTION_URL = + "https://www.elastic.co/guide/en/elasticsearch/plugins/current/creating-classic-plugins.html"; + /** * prints/confirms policy exceptions with the user */ - static void confirmPolicyExceptions(Terminal terminal, Set permissions, boolean batch) throws UserException { - List requested = new ArrayList<>(permissions); + static void confirmPolicyExceptions(Terminal terminal, Set entitlements, boolean batch) throws UserException { + List requested = new ArrayList<>(entitlements); if (requested.isEmpty()) { - terminal.println(Verbosity.VERBOSE, "plugin has a policy file with no additional permissions"); + terminal.println( + Verbosity.NORMAL, + "WARNING: plugin has a policy file with no additional entitlements. Double check this is intentional." + ); } else { - // sort permissions in a reasonable order + // sort entitlements in a reasonable order Collections.sort(requested); if (terminal.isHeadless()) { terminal.errorPrintln( - "WARNING: plugin requires additional permissions: [" + "WARNING: plugin requires additional entitlements: [" + requested.stream().map(each -> '\'' + each + '\'').collect(Collectors.joining(", ")) + "]" ); terminal.errorPrintln( - "See https://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html" - + " for descriptions of what these permissions allow and the associated risks." + "See " + ENTITLEMENTS_DESCRIPTION_URL + " for descriptions of what these entitlements allow and the associated risks." ); } else { terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); - terminal.errorPrintln(Verbosity.NORMAL, "@ WARNING: plugin requires additional permissions @"); + terminal.errorPrintln(Verbosity.NORMAL, "@ WARNING: plugin requires additional entitlements @"); terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); - // print all permissions: - for (String permission : requested) { - terminal.errorPrintln(Verbosity.NORMAL, "* " + permission); + // print all entitlements: + for (String entitlement : requested) { + terminal.errorPrintln(Verbosity.NORMAL, "* " + entitlement); } - terminal.errorPrintln( - Verbosity.NORMAL, - "See https://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html" - ); - terminal.errorPrintln(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks."); + terminal.errorPrintln(Verbosity.NORMAL, "See " + ENTITLEMENTS_DESCRIPTION_URL); + terminal.errorPrintln(Verbosity.NORMAL, "for descriptions of what these entitlements allow and the associated risks."); if (batch == false) { prompt(terminal); @@ -83,53 +77,4 @@ private static void prompt(final Terminal terminal) throws UserException { throw new UserException(ExitCodes.DATA_ERROR, "installation aborted by user"); } } - - /** Format permission type, name, and actions into a string */ - static String formatPermission(Permission permission) { - StringBuilder sb = new StringBuilder(); - - String clazz = null; - if (permission instanceof UnresolvedPermission) { - clazz = ((UnresolvedPermission) permission).getUnresolvedType(); - } else { - clazz = permission.getClass().getName(); - } - sb.append(clazz); - - String name = null; - if (permission instanceof UnresolvedPermission) { - name = ((UnresolvedPermission) permission).getUnresolvedName(); - } else { - name = permission.getName(); - } - if (name != null && name.length() > 0) { - sb.append(' '); - sb.append(name); - } - - String actions = null; - if (permission instanceof UnresolvedPermission) { - actions = ((UnresolvedPermission) permission).getUnresolvedActions(); - } else { - actions = permission.getActions(); - } - if (actions != null && actions.length() > 0) { - sb.append(' '); - sb.append(actions); - } - return sb.toString(); - } - - /** - * Extract a unique set of permissions from the plugin's policy file. Each permission is formatted for output to users. - */ - public static Set getPermissionDescriptions(PluginPolicyInfo pluginPolicyInfo, Path tmpDir) throws IOException { - Set allPermissions = new HashSet<>(PolicyUtil.getPolicyPermissions(null, pluginPolicyInfo.policy(), tmpDir)); - for (URL jar : pluginPolicyInfo.jars()) { - Set jarPermissions = PolicyUtil.getPolicyPermissions(jar, pluginPolicyInfo.policy(), tmpDir); - allPermissions.addAll(jarPermissions); - } - - return allPermissions.stream().map(PluginSecurity::formatPermission).collect(Collectors.toSet()); - } } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java index a8f9e746a24e1..ac9c2b21788c6 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java @@ -93,7 +93,7 @@ private void ensurePluginsNotUsedByOtherPlugins(List plugins) // We build a new map where the keys are plugins that extend plugins // we want to remove and the values are the plugins we can't remove // because of this dependency - Map> pluginDependencyMap = PluginsUtils.getDependencyMapView(env.pluginsFile()); + Map> pluginDependencyMap = PluginsUtils.getDependencyMapView(env.pluginsDir()); for (Map.Entry> entry : pluginDependencyMap.entrySet()) { for (String extendedPlugin : entry.getValue()) { for (InstallablePlugin plugin : plugins) { @@ -121,9 +121,9 @@ private void ensurePluginsNotUsedByOtherPlugins(List plugins) private void checkCanRemove(InstallablePlugin plugin) throws UserException { String pluginId = plugin.getId(); - final Path pluginDir = env.pluginsFile().resolve(pluginId); - final Path pluginConfigDir = env.configFile().resolve(pluginId); - final Path removing = env.pluginsFile().resolve(".removing-" + pluginId); + final Path pluginDir = env.pluginsDir().resolve(pluginId); + final Path pluginConfigDir = env.configDir().resolve(pluginId); + final Path removing = env.pluginsDir().resolve(".removing-" + pluginId); /* * If the plugin does not exist and the plugin config does not exist, fail to the user that the plugin is not found, unless there's @@ -147,7 +147,7 @@ private void checkCanRemove(InstallablePlugin plugin) throws UserException { } } - final Path pluginBinDir = env.binFile().resolve(pluginId); + final Path pluginBinDir = env.binDir().resolve(pluginId); if (Files.exists(pluginBinDir)) { if (Files.isDirectory(pluginBinDir) == false) { throw new UserException(ExitCodes.IO_ERROR, "bin dir for " + pluginId + " is not a directory"); @@ -157,9 +157,9 @@ private void checkCanRemove(InstallablePlugin plugin) throws UserException { private void removePlugin(InstallablePlugin plugin) throws IOException { final String pluginId = plugin.getId(); - final Path pluginDir = env.pluginsFile().resolve(pluginId); - final Path pluginConfigDir = env.configFile().resolve(pluginId); - final Path removing = env.pluginsFile().resolve(".removing-" + pluginId); + final Path pluginDir = env.pluginsDir().resolve(pluginId); + final Path pluginConfigDir = env.configDir().resolve(pluginId); + final Path removing = env.pluginsDir().resolve(".removing-" + pluginId); terminal.println("-> removing [" + pluginId + "]..."); @@ -176,7 +176,7 @@ private void removePlugin(InstallablePlugin plugin) throws IOException { terminal.println(VERBOSE, "removing [" + pluginDir + "]"); } - final Path pluginBinDir = env.binFile().resolve(pluginId); + final Path pluginBinDir = env.binDir().resolve(pluginId); if (Files.exists(pluginBinDir)) { try (Stream paths = Files.list(pluginBinDir)) { pluginPaths.addAll(paths.toList()); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java index d6d0619422770..6d77437bd71d5 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java @@ -61,7 +61,7 @@ public SyncPluginsAction(Terminal terminal, Environment env) { * @throws UserException if a plugins config file is found. */ public static void ensureNoConfigFile(Environment env) throws UserException { - final Path pluginsConfig = env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML); + final Path pluginsConfig = env.configDir().resolve(ELASTICSEARCH_PLUGINS_YML); if (Files.exists(pluginsConfig)) { throw new UserException( ExitCodes.USAGE, @@ -79,16 +79,16 @@ public static void ensureNoConfigFile(Environment env) throws UserException { * @throws Exception if anything goes wrong */ public void execute() throws Exception { - final Path configPath = this.env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML); - final Path previousConfigPath = this.env.pluginsFile().resolve(ELASTICSEARCH_PLUGINS_YML_CACHE); + final Path configPath = this.env.configDir().resolve(ELASTICSEARCH_PLUGINS_YML); + final Path previousConfigPath = this.env.pluginsDir().resolve(ELASTICSEARCH_PLUGINS_YML_CACHE); if (Files.exists(configPath) == false) { // The `PluginsManager` will have checked that this file exists before invoking the action. throw new PluginSyncException("Plugins config does not exist: " + configPath.toAbsolutePath()); } - if (Files.exists(env.pluginsFile()) == false) { - throw new PluginSyncException("Plugins directory missing: " + env.pluginsFile()); + if (Files.exists(env.pluginsDir()) == false) { + throw new PluginSyncException("Plugins directory missing: " + env.pluginsDir()); } // Parse descriptor file @@ -267,14 +267,14 @@ private List getExistingPlugins() throws PluginSyncException { final List plugins = new ArrayList<>(); try { - try (DirectoryStream paths = Files.newDirectoryStream(env.pluginsFile())) { + try (DirectoryStream paths = Files.newDirectoryStream(env.pluginsDir())) { for (Path pluginPath : paths) { String filename = pluginPath.getFileName().toString(); if (filename.startsWith(".")) { continue; } - PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsFile().resolve(pluginPath)); + PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsDir().resolve(pluginPath)); plugins.add(info); // Check for a version mismatch, unless it's an official plugin since we can upgrade them. diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java index 88b24ab9ae614..a5dacebec69bc 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java @@ -37,7 +37,7 @@ public Command create() { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { var action = new SyncPluginsAction(terminal, env); - if (Files.exists(env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML)) == false) { + if (Files.exists(env.configDir().resolve(ELASTICSEARCH_PLUGINS_YML)) == false) { return; } if (Build.current().type() != Build.Type.DOCKER) { diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java index d638534943ecd..a4e503a894ab4 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java @@ -41,11 +41,13 @@ import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.PathUtilsForTesting; import org.elasticsearch.core.Strings; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; +import org.elasticsearch.entitlement.runtime.policy.PolicyUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.plugin.scanner.NamedComponentScanner; @@ -56,6 +58,8 @@ import org.elasticsearch.test.PosixPermissionsResetter; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; import org.elasticsearch.test.jar.JarUtils; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.yaml.YamlXContent; import org.junit.After; import org.junit.Before; @@ -101,6 +105,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; import static org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase.forEachFileRecursively; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -136,8 +141,6 @@ public class InstallPluginActionTests extends ESTestCase { @SuppressForbidden(reason = "sets java.io.tmpdir") public InstallPluginActionTests(FileSystem fs, Function temp) { - assert "false".equals(System.getProperty("tests.security.manager")) : "-Dtests.security.manager=false has to be set"; - this.temp = temp; this.isPosix = fs.supportedFileAttributeViews().contains("posix"); this.isReal = fs == PathUtils.getDefaultFileSystem(); @@ -308,15 +311,20 @@ private static String[] pluginProperties(String name, String[] additionalProps, ).flatMap(Function.identity()).toArray(String[]::new); } - static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException { - StringBuilder securityPolicyContent = new StringBuilder("grant {\n "); - for (String permission : permissions) { - securityPolicyContent.append("permission java.lang.RuntimePermission \""); - securityPolicyContent.append(permission); - securityPolicyContent.append("\";"); + static void writePluginEntitlementPolicy(Path pluginDir, String moduleName, CheckedConsumer policyBuilder) + throws IOException { + try (var builder = YamlXContent.contentBuilder()) { + builder.startObject(); + builder.field(moduleName); + builder.startArray(); + + policyBuilder.accept(builder); + builder.endArray(); + builder.endObject(); + + String policy = org.elasticsearch.common.Strings.toString(builder); + Files.writeString(pluginDir.resolve(PolicyUtils.POLICY_FILE_NAME), policy); } - securityPolicyContent.append("\n};\n"); - Files.write(pluginDir.resolve("plugin-security.policy"), securityPolicyContent.toString().getBytes(StandardCharsets.UTF_8)); } static InstallablePlugin createStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps) @@ -354,7 +362,7 @@ void installPlugins(final List plugins, final Path home, fina } void assertPlugin(String name, Path original, Environment environment) throws IOException { - assertPluginInternal(name, environment.pluginsFile(), original); + assertPluginInternal(name, environment.pluginsDir(), original); assertConfigAndBin(name, original, environment); assertInstallCleaned(environment); } @@ -395,7 +403,7 @@ void assertPluginInternal(String name, Path pluginsFile, Path originalPlugin) th void assertConfigAndBin(String name, Path original, Environment environment) throws IOException { if (Files.exists(original.resolve("bin"))) { - Path binDir = environment.binFile().resolve(name); + Path binDir = environment.binDir().resolve(name); assertTrue("bin dir exists", Files.exists(binDir)); assertTrue("bin is a dir", Files.isDirectory(binDir)); try (DirectoryStream stream = Files.newDirectoryStream(binDir)) { @@ -409,7 +417,7 @@ void assertConfigAndBin(String name, Path original, Environment environment) thr } } if (Files.exists(original.resolve("config"))) { - Path configDir = environment.configFile().resolve(name); + Path configDir = environment.configDir().resolve(name); assertTrue("config dir exists", Files.exists(configDir)); assertTrue("config is a dir", Files.isDirectory(configDir)); @@ -417,7 +425,7 @@ void assertConfigAndBin(String name, Path original, Environment environment) thr GroupPrincipal group = null; if (isPosix) { - PosixFileAttributes configAttributes = Files.getFileAttributeView(environment.configFile(), PosixFileAttributeView.class) + PosixFileAttributes configAttributes = Files.getFileAttributeView(environment.configDir(), PosixFileAttributeView.class) .readAttributes(); user = configAttributes.owner(); group = configAttributes.group(); @@ -446,7 +454,7 @@ void assertConfigAndBin(String name, Path original, Environment environment) thr } void assertInstallCleaned(Environment environment) throws IOException { - try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsFile())) { + try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsDir())) { for (Path file : stream) { if (file.getFileName().toString().startsWith(".installing")) { fail("Installation dir still exists, " + file); @@ -549,7 +557,7 @@ public void testTransaction() throws Exception { () -> installPlugins(List.of(pluginZip, nonexistentPluginZip), env.v1()) ); assertThat(e.getMessage(), containsString("does-not-exist")); - final Path fakeInstallPath = env.v2().pluginsFile().resolve("fake"); + final Path fakeInstallPath = env.v2().pluginsDir().resolve("fake"); // fake should have been removed when the file not found exception occurred assertFalse(Files.exists(fakeInstallPath)); assertInstallCleaned(env.v2()); @@ -557,7 +565,7 @@ public void testTransaction() throws Exception { public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception { InstallablePlugin pluginZip = createPluginZip("fake", pluginDir); - final Path removing = env.v2().pluginsFile().resolve(".removing-failed"); + final Path removing = env.v2().pluginsDir().resolve(".removing-failed"); Files.createDirectory(removing); final IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip)); final String expected = Strings.format( @@ -603,11 +611,11 @@ public void testUnknownPlugin() { public void testPluginsDirReadOnly() throws Exception { assumeTrue("posix and filesystem", isPosix && isReal); - try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsFile())) { + try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsDir())) { pluginsAttrs.setPermissions(new HashSet<>()); InstallablePlugin pluginZip = createPluginZip("fake", pluginDir); IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip)); - assertThat(e.getMessage(), containsString(env.v2().pluginsFile().toString())); + assertThat(e.getMessage(), containsString(env.v2().pluginsDir().toString())); } assertInstallCleaned(env.v2()); } @@ -694,7 +702,7 @@ public void testBinConflict() throws Exception { Files.createFile(binDir.resolve("somescript")); InstallablePlugin pluginZip = createPluginZip("elasticsearch", pluginDir); FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip)); - assertThat(e.getMessage(), containsString(env.v2().binFile().resolve("elasticsearch").toString())); + assertThat(e.getMessage(), containsString(env.v2().binDir().resolve("elasticsearch").toString())); assertInstallCleaned(env.v2()); } @@ -704,7 +712,7 @@ public void testBinPermissions() throws Exception { Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); InstallablePlugin pluginZip = createPluginZip("fake", pluginDir); - try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) { + try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binDir())) { Set perms = binAttrs.getCopyPermissions(); // make sure at least one execute perm is missing, so we know we forced it during installation perms.remove(PosixFilePermission.GROUP_EXECUTE); @@ -734,7 +742,7 @@ public void testPluginPermissions() throws Exception { installPlugin(pluginZip); assertPlugin("fake", tempPluginDir, env.v2()); - final Path fake = env.v2().pluginsFile().resolve("fake"); + final Path fake = env.v2().pluginsDir().resolve("fake"); final Path resources = fake.resolve("resources"); final Path platform = fake.resolve("platform"); final Path platformName = platform.resolve("linux-x86_64"); @@ -784,12 +792,12 @@ public void testConfig() throws Exception { } public void testExistingConfig() throws Exception { - Path envConfigDir = env.v2().configFile().resolve("fake"); + Path envConfigDir = env.v2().configDir().resolve("fake"); Files.createDirectories(envConfigDir); - Files.write(envConfigDir.resolve("custom.yml"), "existing config".getBytes(StandardCharsets.UTF_8)); + Files.writeString(envConfigDir.resolve("custom.yml"), "existing config"); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); - Files.write(configDir.resolve("custom.yml"), "new config".getBytes(StandardCharsets.UTF_8)); + Files.writeString(configDir.resolve("custom.yml"), "new config"); Files.createFile(configDir.resolve("other.yml")); InstallablePlugin pluginZip = createPluginZip("fake", pluginDir); installPlugin(pluginZip); @@ -892,7 +900,7 @@ public void testInstallMisspelledOfficialPlugins() { public void testBatchFlag() throws Exception { installPlugin(true); - assertThat(terminal.getErrorOutput(), containsString("WARNING: plugin requires additional permissions")); + assertThat(terminal.getErrorOutput(), containsString("WARNING: plugin requires additional entitlements")); assertThat(terminal.getOutput(), containsString("-> Downloading")); // No progress bar in batch mode assertThat(terminal.getOutput(), not(containsString("100%"))); @@ -921,7 +929,7 @@ public void testPluginAlreadyInstalled() throws Exception { e.getMessage(), equalTo( "plugin directory [" - + env.v2().pluginsFile().resolve("fake") + + env.v2().pluginsDir().resolve("fake") + "] already exists; " + "if you need to update the plugin, uninstall it first using command 'remove fake'" ) @@ -940,12 +948,12 @@ public void testPluginHasDifferentNameThatDescriptor() throws Exception { assertThat(e.getMessage(), equalTo("Expected downloaded plugin to have ID [other-fake] but found [fake]")); } - private void installPlugin(boolean isBatch, String... additionalProperties) throws Exception { - // if batch is enabled, we also want to add a security policy + private void installPlugin(boolean isBatch) throws Exception { + // if batch is enabled, we also want to add an entitlement policy if (isBatch) { - writePluginSecurityPolicy(pluginDir, "setFactory"); + writePluginEntitlementPolicy(pluginDir, ALL_UNNAMED, builder -> builder.value("manage_threads")); } - InstallablePlugin pluginZip = createPlugin("fake", pluginDir, additionalProperties); + InstallablePlugin pluginZip = createPlugin("fake", pluginDir); skipJarHellAction.setEnvironment(env.v2()); skipJarHellAction.setBatch(isBatch); skipJarHellAction.execute(List.of(pluginZip)); @@ -1031,13 +1039,13 @@ URL openUrl(String urlString) throws IOException { Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension); byte[] zipbytes = Files.readAllBytes(pluginZipPath); String checksum = shaCalculator.apply(zipbytes); - Files.write(shaFile, checksum.getBytes(StandardCharsets.UTF_8)); + Files.writeString(shaFile, checksum); return shaFile.toUri().toURL(); } else if ((url + ".asc").equals(urlString)) { final Path ascFile = temp.apply("asc").resolve("downloaded.zip" + ".asc"); final byte[] zipBytes = Files.readAllBytes(pluginZipPath); final String asc = signature.apply(zipBytes, secretKey); - Files.write(ascFile, asc.getBytes(StandardCharsets.UTF_8)); + Files.writeString(ascFile, asc); return ascFile.toUri().toURL(); } return null; @@ -1499,7 +1507,7 @@ private void assertPolicyConfirmation(Tuple pathEnvironmentTu assertThat(e.getMessage(), containsString("installation aborted by user")); assertThat(terminal.getErrorOutput(), containsString("WARNING: " + warning)); - try (Stream fileStream = Files.list(pathEnvironmentTuple.v2().pluginsFile())) { + try (Stream fileStream = Files.list(pathEnvironmentTuple.v2().pluginsDir())) { assertThat(fileStream.collect(Collectors.toList()), empty()); } @@ -1512,7 +1520,7 @@ private void assertPolicyConfirmation(Tuple pathEnvironmentTu e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); assertThat(e.getMessage(), containsString("installation aborted by user")); assertThat(terminal.getErrorOutput(), containsString("WARNING: " + warning)); - try (Stream fileStream = Files.list(pathEnvironmentTuple.v2().pluginsFile())) { + try (Stream fileStream = Files.list(pathEnvironmentTuple.v2().pluginsDir())) { assertThat(fileStream.collect(Collectors.toList()), empty()); } } @@ -1529,10 +1537,13 @@ private void assertPolicyConfirmation(Tuple pathEnvironmentTu } public void testPolicyConfirmation() throws Exception { - writePluginSecurityPolicy(pluginDir, "getClassLoader", "setFactory"); + writePluginEntitlementPolicy(pluginDir, "test.plugin.module", builder -> { + builder.value("manage_threads"); + builder.value("outbound_network"); + }); InstallablePlugin pluginZip = createPluginZip("fake", pluginDir); - assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions"); + assertPolicyConfirmation(env, pluginZip, "plugin requires additional entitlements"); assertPlugin("fake", pluginDir, env.v2()); } @@ -1566,7 +1577,7 @@ public void testStablePluginWithNamedComponentsFile() throws Exception { InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, true); installPlugins(List.of(stablePluginZip), env.v1()); assertPlugin("stable1", pluginDir, env.v2()); - assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON()); + assertNamedComponentFile("stable1", env.v2().pluginsDir(), namedComponentsJSON()); } @SuppressWarnings("unchecked") @@ -1577,7 +1588,7 @@ public void testStablePluginWithoutNamedComponentsFile() throws Exception { installPlugins(List.of(stablePluginZip), env.v1()); assertPlugin("stable1", pluginDir, env.v2()); - assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON()); + assertNamedComponentFile("stable1", env.v2().pluginsDir(), namedComponentsJSON()); } public void testGetSemanticVersion() { diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java index 0064b8c4bc513..5249aeefc2f2d 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java @@ -65,7 +65,7 @@ private static void buildFakePlugin( final boolean hasNativeController ) throws IOException { PluginTestUtil.writePluginProperties( - env.pluginsFile().resolve(name), + env.pluginsDir().resolve(name), "description", description, "name", @@ -84,9 +84,9 @@ private static void buildFakePlugin( } public void testPluginsDirMissing() throws Exception { - Files.delete(env.pluginsFile()); + Files.delete(env.pluginsDir()); IOException e = expectThrows(IOException.class, () -> execute()); - assertEquals("Plugins directory missing: " + env.pluginsFile(), e.getMessage()); + assertEquals("Plugins directory missing: " + env.pluginsDir(), e.getMessage()); } public void testNoPlugins() throws Exception { @@ -112,7 +112,7 @@ public void testPluginWithVerbose() throws Exception { execute("-v"); assertEquals( buildMultiline( - "Plugins directory: " + env.pluginsFile(), + "Plugins directory: " + env.pluginsDir(), "fake_plugin", "- Plugin information:", "Name: fake_plugin", @@ -134,7 +134,7 @@ public void testPluginWithNativeController() throws Exception { execute("-v"); assertEquals( buildMultiline( - "Plugins directory: " + env.pluginsFile(), + "Plugins directory: " + env.pluginsDir(), "fake_plugin1", "- Plugin information:", "Name: fake_plugin1", @@ -157,7 +157,7 @@ public void testPluginWithVerboseMultiplePlugins() throws Exception { execute("-v"); assertEquals( buildMultiline( - "Plugins directory: " + env.pluginsFile(), + "Plugins directory: " + env.pluginsDir(), "fake_plugin1", "- Plugin information:", "Name: fake_plugin1", @@ -193,14 +193,14 @@ public void testPluginWithoutVerboseMultiplePlugins() throws Exception { } public void testPluginWithoutDescriptorFile() throws Exception { - final Path pluginDir = env.pluginsFile().resolve("fake1"); + final Path pluginDir = env.pluginsDir().resolve("fake1"); Files.createDirectories(pluginDir); var e = expectThrows(IllegalStateException.class, () -> execute()); assertThat(e.getMessage(), equalTo("Plugin [fake1] is missing a descriptor properties file.")); } public void testPluginWithWrongDescriptorFile() throws Exception { - final Path pluginDir = env.pluginsFile().resolve("fake1"); + final Path pluginDir = env.pluginsDir().resolve("fake1"); PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc"); var e = expectThrows(IllegalArgumentException.class, () -> execute()); assertThat(e.getMessage(), startsWith("property [name] is missing for plugin")); @@ -208,7 +208,7 @@ public void testPluginWithWrongDescriptorFile() throws Exception { public void testExistingIncompatiblePlugin() throws Exception { PluginTestUtil.writePluginProperties( - env.pluginsFile().resolve("fake_plugin1"), + env.pluginsDir().resolve("fake_plugin1"), "description", "fake desc 1", "name", diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java index aabdd4aaceb9e..8338c395e5e4c 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java @@ -58,11 +58,11 @@ public void setUp() throws Exception { } void createPlugin(String name) throws IOException { - createPlugin(env.pluginsFile(), name, Version.CURRENT); + createPlugin(env.pluginsDir(), name, Version.CURRENT); } void createPlugin(String name, Version version) throws IOException { - createPlugin(env.pluginsFile(), name, version); + createPlugin(env.pluginsDir(), name, version); } void createPlugin(Path path, String name, Version version) throws IOException { @@ -98,7 +98,7 @@ static MockTerminal removePlugin(List pluginIds, Path home, boolean purg } static void assertRemoveCleaned(Environment env) throws IOException { - try (DirectoryStream stream = Files.newDirectoryStream(env.pluginsFile())) { + try (DirectoryStream stream = Files.newDirectoryStream(env.pluginsDir())) { for (Path file : stream) { if (file.getFileName().toString().startsWith(".removing")) { fail("Removal dir still exists, " + file); @@ -115,84 +115,84 @@ public void testMissing() throws Exception { public void testBasic() throws Exception { createPlugin("fake"); - Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar")); - Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir")); + Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar")); + Files.createDirectory(env.pluginsDir().resolve("fake").resolve("subdir")); createPlugin("other"); removePlugin("fake", home, randomBoolean()); - assertFalse(Files.exists(env.pluginsFile().resolve("fake"))); - assertTrue(Files.exists(env.pluginsFile().resolve("other"))); + assertFalse(Files.exists(env.pluginsDir().resolve("fake"))); + assertTrue(Files.exists(env.pluginsDir().resolve("other"))); assertRemoveCleaned(env); } /** Check that multiple plugins can be removed at the same time. */ public void testRemoveMultiple() throws Exception { createPlugin("fake"); - Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar")); - Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir")); + Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar")); + Files.createDirectory(env.pluginsDir().resolve("fake").resolve("subdir")); createPlugin("other"); - Files.createFile(env.pluginsFile().resolve("other").resolve("plugin.jar")); - Files.createDirectory(env.pluginsFile().resolve("other").resolve("subdir")); + Files.createFile(env.pluginsDir().resolve("other").resolve("plugin.jar")); + Files.createDirectory(env.pluginsDir().resolve("other").resolve("subdir")); removePlugin("fake", home, randomBoolean()); removePlugin("other", home, randomBoolean()); - assertFalse(Files.exists(env.pluginsFile().resolve("fake"))); - assertFalse(Files.exists(env.pluginsFile().resolve("other"))); + assertFalse(Files.exists(env.pluginsDir().resolve("fake"))); + assertFalse(Files.exists(env.pluginsDir().resolve("other"))); assertRemoveCleaned(env); } public void testBin() throws Exception { createPlugin("fake"); - Path binDir = env.binFile().resolve("fake"); + Path binDir = env.binDir().resolve("fake"); Files.createDirectories(binDir); Files.createFile(binDir.resolve("somescript")); removePlugin("fake", home, randomBoolean()); - assertFalse(Files.exists(env.pluginsFile().resolve("fake"))); - assertTrue(Files.exists(env.binFile().resolve("elasticsearch"))); + assertFalse(Files.exists(env.pluginsDir().resolve("fake"))); + assertTrue(Files.exists(env.binDir().resolve("elasticsearch"))); assertFalse(Files.exists(binDir)); assertRemoveCleaned(env); } public void testBinNotDir() throws Exception { createPlugin("fake"); - Files.createFile(env.binFile().resolve("fake")); + Files.createFile(env.binDir().resolve("fake")); UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, randomBoolean())); assertThat(e.getMessage(), containsString("not a directory")); - assertTrue(Files.exists(env.pluginsFile().resolve("fake"))); // did not remove - assertTrue(Files.exists(env.binFile().resolve("fake"))); + assertTrue(Files.exists(env.pluginsDir().resolve("fake"))); // did not remove + assertTrue(Files.exists(env.binDir().resolve("fake"))); assertRemoveCleaned(env); } public void testConfigDirPreserved() throws Exception { createPlugin("fake"); - final Path configDir = env.configFile().resolve("fake"); + final Path configDir = env.configDir().resolve("fake"); Files.createDirectories(configDir); Files.createFile(configDir.resolve("fake.yml")); final MockTerminal terminal = removePlugin("fake", home, false); - assertTrue(Files.exists(env.configFile().resolve("fake"))); + assertTrue(Files.exists(env.configDir().resolve("fake"))); assertThat(terminal.getOutput(), containsString(expectedConfigDirPreservedMessage(configDir))); assertRemoveCleaned(env); } public void testPurgePluginExists() throws Exception { createPlugin("fake"); - final Path configDir = env.configFile().resolve("fake"); + final Path configDir = env.configDir().resolve("fake"); if (randomBoolean()) { Files.createDirectories(configDir); Files.createFile(configDir.resolve("fake.yml")); } final MockTerminal terminal = removePlugin("fake", home, true); - assertFalse(Files.exists(env.configFile().resolve("fake"))); + assertFalse(Files.exists(env.configDir().resolve("fake"))); assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir)))); assertRemoveCleaned(env); } public void testPurgePluginDoesNotExist() throws Exception { - final Path configDir = env.configFile().resolve("fake"); + final Path configDir = env.configDir().resolve("fake"); Files.createDirectories(configDir); Files.createFile(configDir.resolve("fake.yml")); final MockTerminal terminal = removePlugin("fake", home, true); - assertFalse(Files.exists(env.configFile().resolve("fake"))); + assertFalse(Files.exists(env.configDir().resolve("fake"))); assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir)))); assertRemoveCleaned(env); } @@ -203,8 +203,8 @@ public void testPurgeNothingExists() throws Exception { } public void testPurgeOnlyMarkerFileExists() throws Exception { - final Path configDir = env.configFile().resolve("fake"); - final Path removing = env.pluginsFile().resolve(".removing-fake"); + final Path configDir = env.configDir().resolve("fake"); + final Path removing = env.pluginsDir().resolve(".removing-fake"); Files.createFile(removing); final MockTerminal terminal = removePlugin("fake", home, randomBoolean()); assertFalse(Files.exists(removing)); @@ -213,7 +213,7 @@ public void testPurgeOnlyMarkerFileExists() throws Exception { public void testNoConfigDirPreserved() throws Exception { createPlugin("fake"); - final Path configDir = env.configFile().resolve("fake"); + final Path configDir = env.configDir().resolve("fake"); final MockTerminal terminal = removePlugin("fake", home, randomBoolean()); assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir)))); } @@ -250,8 +250,8 @@ public void testMissingPluginName() { public void testRemoveWhenRemovingMarker() throws Exception { createPlugin("fake"); - Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar")); - Files.createFile(env.pluginsFile().resolve(".removing-fake")); + Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar")); + Files.createFile(env.pluginsDir().resolve(".removing-fake")); removePlugin("fake", home, randomBoolean()); } @@ -262,10 +262,10 @@ public void testRemoveWhenRemovingMarker() throws Exception { public void testRemoveMigratedPluginsWhenInstalled() throws Exception { for (String id : List.of("repository-azure", "repository-gcs", "repository-s3")) { createPlugin(id); - Files.createFile(env.pluginsFile().resolve(id).resolve("plugin.jar")); + Files.createFile(env.pluginsDir().resolve(id).resolve("plugin.jar")); final MockTerminal terminal = removePlugin(id, home, randomBoolean()); - assertThat(Files.exists(env.pluginsFile().resolve(id)), is(false)); + assertThat(Files.exists(env.pluginsDir().resolve(id)), is(false)); // This message shouldn't be printed if plugin was actually installed. assertThat(terminal.getErrorOutput(), not(containsString("plugin [" + id + "] is no longer a plugin"))); } @@ -288,11 +288,11 @@ public void testRemoveMigratedPluginsWhenNotInstalled() throws Exception { */ public void testRemoveRegularInstalledPluginAndMigratedUninstalledPlugin() throws Exception { createPlugin("fake"); - Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar")); + Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar")); final MockTerminal terminal = removePlugin(List.of("fake", "repository-s3"), home, randomBoolean()); - assertThat(Files.exists(env.pluginsFile().resolve("fake")), is(false)); + assertThat(Files.exists(env.pluginsDir().resolve("fake")), is(false)); assertThat(terminal.getErrorOutput(), containsString("plugin [repository-s3] is no longer a plugin")); } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java index 2d2336428a0a5..1a09736160956 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java @@ -55,10 +55,10 @@ public void setUp() throws Exception { Path home = createTempDir(); Settings settings = Settings.builder().put("path.home", home).build(); env = TestEnvironment.newEnvironment(settings); - Files.createDirectories(env.binFile()); - Files.createFile(env.binFile().resolve("elasticsearch")); - Files.createDirectories(env.configFile()); - Files.createDirectories(env.pluginsFile()); + Files.createDirectories(env.binDir()); + Files.createFile(env.binDir().resolve("elasticsearch")); + Files.createDirectories(env.configDir()); + Files.createDirectories(env.pluginsDir()); terminal = MockTerminal.create(); action = new SyncPluginsAction(terminal, env); @@ -78,7 +78,7 @@ public void test_ensureNoConfigFile_withoutConfig_doesNothing() throws Exception * then an exception is thrown. */ public void test_ensureNoConfigFile_withConfig_throwsException() throws Exception { - Files.createFile(env.configFile().resolve("elasticsearch-plugins.yml")); + Files.createFile(env.configDir().resolve("elasticsearch-plugins.yml")); final UserException e = expectThrows(UserException.class, () -> SyncPluginsAction.ensureNoConfigFile(env)); assertThat(e.getMessage(), Matchers.matchesPattern("^Plugins config \\[.*] exists.*$")); @@ -354,7 +354,7 @@ private void createPlugin(String name) throws IOException { private void createPlugin(String name, String version) throws IOException { PluginTestUtil.writePluginProperties( - env.pluginsFile().resolve(name), + env.pluginsDir().resolve(name), "description", "dummy", "name", diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java index 9430cb598cf02..2ae58040437af 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java @@ -24,7 +24,7 @@ public class KeyStoreLoader implements SecureSettingsLoader { @Override public LoadedSecrets load(Environment environment, Terminal terminal) throws Exception { // See if we have a keystore already present - KeyStoreWrapper secureSettings = KeyStoreWrapper.load(environment.configFile()); + KeyStoreWrapper secureSettings = KeyStoreWrapper.load(environment.configDir()); // If there's no keystore or the keystore has no password, set an empty password var password = (secureSettings == null || secureSettings.hasPassword() == false) ? new SecureString(new char[0]) @@ -35,7 +35,7 @@ public LoadedSecrets load(Environment environment, Terminal terminal) throws Exc @Override public SecureSettings bootstrap(Environment environment, SecureString password) throws Exception { - return KeyStoreWrapper.bootstrap(environment.configFile(), () -> password); + return KeyStoreWrapper.bootstrap(environment.configDir(), () -> password); } @Override diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java index 22b62972befe4..be454350133eb 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java @@ -150,7 +150,7 @@ private void validateConfig(OptionSet options, Environment env) throws UserExcep throw new UserException(ExitCodes.USAGE, "Multiple --enrollment-token parameters are not allowed"); } - Path log4jConfig = env.configFile().resolve("log4j2.properties"); + Path log4jConfig = env.configDir().resolve("log4j2.properties"); if (Files.exists(log4jConfig) == false) { throw new UserException(ExitCodes.CONFIG, "Missing logging config file at " + log4jConfig); } @@ -239,7 +239,7 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings } validatePidFile(pidFile); } - return new ServerArgs(daemonize, quiet, pidFile, secrets, env.settings(), env.configFile(), env.logsFile()); + return new ServerArgs(daemonize, quiet, pidFile, secrets, env.settings(), env.configDir(), env.logsDir()); } @Override diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java index 293c4af3270b9..adebf6be9842b 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java @@ -109,6 +109,7 @@ private List getJvmArgs() { esHome.resolve("lib").toString(), // Special circumstances require some modules (not depended on by the main server module) to be explicitly added: "--add-modules=jdk.net", // needed to reflectively set extended socket options + "--add-modules=jdk.management.agent", // needed by external debug tools to grab thread and heap dumps // we control the module path, which may have additional modules not required by server "--add-modules=ALL-MODULE-PATH", "-m", diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 21294f1c5f01b..3bab8d528ffef 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -11,8 +11,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.jdk.RuntimeVersionFeature; import java.io.IOException; @@ -27,10 +25,10 @@ final class SystemJvmOptions { static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { String distroType = sysprops.get("es.distribution.type"); + String javaType = sysprops.get("es.java.type"); boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot"); - boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false")); - // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly - boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled; + + boolean useEntitlements = true; return Stream.of( Stream.of( /* @@ -67,8 +65,11 @@ static List systemJvmOptions(Settings nodeSettings, final Map TEST_SYSPROPS = Map.of("os.name", "Linux", "os.arch", "aarch64"); + private static final Path ENTITLEMENTS_LIB_DIR = Path.of("lib", "entitlement-bridge"); + + @BeforeClass + public static void beforeClass() throws IOException { + Files.createDirectories(ENTITLEMENTS_LIB_DIR); + Files.createTempFile(ENTITLEMENTS_LIB_DIR, "mock-entitlements-bridge", ".jar"); + } + + @AfterClass + public static void afterClass() throws IOException { + IOUtils.rm(Path.of("lib")); + } + public void testSubstitution() { final List jvmOptions = JvmOptionsParser.substitutePlaceholders( List.of("-Djava.io.tmpdir=${ES_TMPDIR}"), diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java index bacc89548c9a1..41ed786690328 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java @@ -185,7 +185,7 @@ public void testElasticsearchSettingCanNotBeEmpty() throws Exception { } public void testElasticsearchSettingCanNotBeDuplicated() throws Exception { - assertUsage(containsString("setting [foo] already set, saw [bar] and [baz]"), "-E", "foo=bar", "-E", "foo=baz"); + assertUsage(containsString("setting [foo] set twice via command line -E"), "-E", "foo=bar", "-E", "foo=baz"); } public void testUnknownOption() throws Exception { diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java index 66ee712fcce95..2854d76c110d1 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java @@ -43,8 +43,8 @@ class WindowsServiceDaemon extends EnvironmentAwareCommand { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { // the Windows service daemon doesn't support secure settings implementations other than the keystore - try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))) { - var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configFile(), env.logsFile()); + try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configDir(), () -> new SecureString(new char[0]))) { + var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configDir(), env.logsDir()); var tempDir = ServerProcessUtils.setupTempDir(processInfo); var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir, new MachineDependentHeap()); var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) diff --git a/docs/build.gradle b/docs/build.gradle index cdb879485ae3c..6ac29ef0b2469 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.doc.DocSnippetTask import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT @@ -37,7 +38,7 @@ ext.docsFileTree = fileTree(projectDir) { } tasks.named("yamlRestTest") { - if (buildParams.isSnapshotBuild() == false) { + if (buildParams.snapshotBuild == false) { // LOOKUP is not available in snapshots systemProperty 'tests.rest.blacklist', [ "reference/esql/processing-commands/lookup/esql-lookup-example" @@ -82,7 +83,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' keystorePassword 'keystore-password' - if (buildParams.isSnapshotBuild() == false) { + if (buildParams.snapshotBuild == false) { requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0) } } @@ -119,6 +120,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 systemProperty 'es.transport.cname_in_publish_address', 'true' + systemProperty 'es.queryable_built_in_roles_enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") @@ -2047,3 +2049,29 @@ setups['setup-snapshots'] = setups['setup-repository'] + ''' setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events) } } + +tasks.register('verifyDocsLuceneVersion') { + doFirst { + File docsVersionsFile = file('Versions.asciidoc') + List versionLines = docsVersionsFile.readLines('UTF-8') + String docsLuceneVersion = null + for (String line : versionLines) { + if (line.startsWith(':lucene_version:')) { + docsLuceneVersion = line.split()[1] + } + } + if (docsLuceneVersion == null) { + throw new GradleException('Could not find lucene version in docs version file') + } + String expectedLuceneVersion = VersionProperties.lucene + // remove potential -snapshot-{gitrev} suffix + expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/ + if (docsLuceneVersion != expectedLuceneVersion) { + throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]") + } + } +} + +tasks.named('check') { + dependsOn 'verifyDocsLuceneVersion' +} diff --git a/docs/changelog/117176.yaml b/docs/changelog/117176.yaml new file mode 100644 index 0000000000000..26e0d3635bc9e --- /dev/null +++ b/docs/changelog/117176.yaml @@ -0,0 +1,5 @@ +pr: 117176 +summary: Integrate IBM watsonx to Inference API for re-ranking task +area: Experiences +type: enhancement +issues: [] diff --git a/docs/changelog/117201.yaml b/docs/changelog/117201.yaml deleted file mode 100644 index f8a2be35c70a3..0000000000000 --- a/docs/changelog/117201.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117201 -summary: "Use `field_caps` native nested fields filtering" -area: ES|QL -type: bug -issues: - - 117054 diff --git a/docs/changelog/117504.yaml b/docs/changelog/117504.yaml deleted file mode 100644 index 91a62c61b88f2..0000000000000 --- a/docs/changelog/117504.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117504 -summary: Fix NPE caused by race condition in async search when minimise round trips - is true -area: Search -type: bug -issues: [] diff --git a/docs/changelog/119743.yaml b/docs/changelog/119743.yaml index b6f53c0dd1aed..850c48347260a 100644 --- a/docs/changelog/119743.yaml +++ b/docs/changelog/119743.yaml @@ -1,5 +1,5 @@ pr: 119743 -summary: POC mark read-only +summary: Enhance add-block API to flush and add 'verified' metadata area: Engine type: enhancement issues: [] diff --git a/docs/changelog/119748.yaml b/docs/changelog/119748.yaml deleted file mode 100644 index 8b29fb7c1a647..0000000000000 --- a/docs/changelog/119748.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 119748 -summary: Issue S3 web identity token refresh call with sufficient permissions -area: Snapshot/Restore -type: bug -issues: - - 119747 diff --git a/docs/changelog/119889.yaml b/docs/changelog/119889.yaml new file mode 100644 index 0000000000000..e07d8643e379c --- /dev/null +++ b/docs/changelog/119889.yaml @@ -0,0 +1,5 @@ +pr: 119889 +summary: Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` +area: "ES|QL" +type: enhancement +issues: [] diff --git a/docs/changelog/120014.yaml b/docs/changelog/120014.yaml deleted file mode 100644 index bef1f3ba49939..0000000000000 --- a/docs/changelog/120014.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 120014 -summary: Fix potential file leak in ES816BinaryQuantizedVectorsWriter -area: Search -type: bug -issues: - - 119981 diff --git a/docs/changelog/120062.yaml b/docs/changelog/120062.yaml deleted file mode 100644 index 42e8d97f17444..0000000000000 --- a/docs/changelog/120062.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 120062 -summary: Update Text Similarity Reranker to Properly Handle Aliases -area: Ranking -type: bug -issues: - - 119617 diff --git a/docs/changelog/120133.yaml b/docs/changelog/120133.yaml deleted file mode 100644 index 4ec88267a1bf8..0000000000000 --- a/docs/changelog/120133.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 120133 -summary: Use approximation to advance matched queries -area: Search -type: bug -issues: - - 120130 diff --git a/docs/changelog/120256.yaml b/docs/changelog/120256.yaml deleted file mode 100644 index c4ee5ab1705c5..0000000000000 --- a/docs/changelog/120256.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 120256 -summary: Improve memory aspects of enrich cache -area: Ingest Node -type: enhancement -issues: - - 96050 - - 120021 diff --git a/docs/changelog/120483.yaml b/docs/changelog/120483.yaml deleted file mode 100644 index 20da3b9ab4e8d..0000000000000 --- a/docs/changelog/120483.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 120483 -summary: Fix NPE on disabled API auth key cache -area: Authentication -type: bug -issues: [] diff --git a/docs/changelog/120590.yaml b/docs/changelog/120590.yaml deleted file mode 100644 index 56abe44fbce1e..0000000000000 --- a/docs/changelog/120590.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 120590 -summary: Map `scope.name` as a dimension -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/120717.yaml b/docs/changelog/120717.yaml deleted file mode 100644 index c5609e7e3df5f..0000000000000 --- a/docs/changelog/120717.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 120717 -summary: Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled -area: Ranking -type: bug -issues: - - 120739 diff --git a/docs/changelog/120781.yaml b/docs/changelog/120781.yaml deleted file mode 100644 index 67c7d90528d6e..0000000000000 --- a/docs/changelog/120781.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 120781 -summary: Add back `keep_alive` to `async_search.submit` rest-api-spec -area: Search -type: bug -issues: [] diff --git a/docs/changelog/120809.yaml b/docs/changelog/120809.yaml deleted file mode 100644 index 30a3736dc93a4..0000000000000 --- a/docs/changelog/120809.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 120809 -summary: LTR sometines throw `NullPointerException:` Cannot read field "approximation" - because "top" is null -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/120852.yaml b/docs/changelog/120852.yaml new file mode 100644 index 0000000000000..90a05aa860f3f --- /dev/null +++ b/docs/changelog/120852.yaml @@ -0,0 +1,5 @@ +pr: 120852 +summary: Correct line and column numbers of missing named parameters +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/120930.yaml b/docs/changelog/120930.yaml deleted file mode 100644 index 376edb7632a0b..0000000000000 --- a/docs/changelog/120930.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 120930 -summary: Normalize negative scores for `text_similarity_reranker` retriever -area: Ranking -type: bug -issues: - - 120201 diff --git a/docs/changelog/121120.yaml b/docs/changelog/121120.yaml deleted file mode 100644 index ff375d9088ac9..0000000000000 --- a/docs/changelog/121120.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 121120 -summary: Revert "Reduce Data Loss in System Indices Migration 8x" -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/121156.yaml b/docs/changelog/121156.yaml new file mode 100644 index 0000000000000..8f9c5ccdb38c9 --- /dev/null +++ b/docs/changelog/121156.yaml @@ -0,0 +1,5 @@ +pr: 121156 +summary: Remove redundant sorts from execution plan +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/121193.yaml b/docs/changelog/121193.yaml new file mode 100644 index 0000000000000..af45b0656265f --- /dev/null +++ b/docs/changelog/121193.yaml @@ -0,0 +1,18 @@ +pr: 121193 +summary: Enable LOOKUP JOIN in non-snapshot builds +area: ES|QL +type: enhancement +issues: + - 121185 +highlight: + title: Enable LOOKUP JOIN in non-snapshot builds + body: |- + This effectively releases LOOKUP JOIN into tech preview. Docs will + follow in a separate PR. + + - Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds. + - Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages. + - Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries. + + Also fix https://github.com/elastic/elasticsearch/issues/121185 + notable: true diff --git a/docs/changelog/121196.yaml b/docs/changelog/121196.yaml new file mode 100644 index 0000000000000..f5168b1ea436c --- /dev/null +++ b/docs/changelog/121196.yaml @@ -0,0 +1,5 @@ +pr: 121196 +summary: Fix geoip databases index access after system feature migration +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/121325.yaml b/docs/changelog/121325.yaml new file mode 100644 index 0000000000000..9a9edc67d19fa --- /dev/null +++ b/docs/changelog/121325.yaml @@ -0,0 +1,5 @@ +pr: 121325 +summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality' +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/121392.yaml b/docs/changelog/121392.yaml new file mode 100644 index 0000000000000..6323789f071d8 --- /dev/null +++ b/docs/changelog/121392.yaml @@ -0,0 +1,5 @@ +pr: 121392 +summary: Include data streams when converting an existing resource to a system resource +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/121396.yaml b/docs/changelog/121396.yaml new file mode 100644 index 0000000000000..1d77a8fbb0079 --- /dev/null +++ b/docs/changelog/121396.yaml @@ -0,0 +1,5 @@ +pr: 121396 +summary: Change format for Unified Chat +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/121552.yaml b/docs/changelog/121552.yaml new file mode 100644 index 0000000000000..c12e7615d1245 --- /dev/null +++ b/docs/changelog/121552.yaml @@ -0,0 +1,5 @@ +pr: 121552 +summary: Fix a bug in TOP +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/121720.yaml b/docs/changelog/121720.yaml new file mode 100644 index 0000000000000..40dcfaeb770dd --- /dev/null +++ b/docs/changelog/121720.yaml @@ -0,0 +1,5 @@ +pr: 121720 +summary: Skip fetching _inference_fields field in legacy semantic_text format +area: Search +type: bug +issues: [] diff --git a/docs/changelog/121727.yaml b/docs/changelog/121727.yaml new file mode 100644 index 0000000000000..80c0a5eae4335 --- /dev/null +++ b/docs/changelog/121727.yaml @@ -0,0 +1,7 @@ +pr: 121727 +summary: Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` +area: Downsampling +type: bug +issues: + - 119696 + - 96076 diff --git a/docs/changelog/121821.yaml b/docs/changelog/121821.yaml new file mode 100644 index 0000000000000..1e8edd09dcd9a --- /dev/null +++ b/docs/changelog/121821.yaml @@ -0,0 +1,6 @@ +pr: 121821 +summary: Fix get all inference endponts not returning multiple endpoints sharing model + deployment +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/121843.yaml b/docs/changelog/121843.yaml new file mode 100644 index 0000000000000..85b19e317a09c --- /dev/null +++ b/docs/changelog/121843.yaml @@ -0,0 +1,6 @@ +pr: 121843 +summary: Fix async stop sometimes not properly collecting result +area: ES|QL +type: bug +issues: + - 121249 diff --git a/docs/changelog/121850.yaml b/docs/changelog/121850.yaml new file mode 100644 index 0000000000000..b6c5ba2e03fe8 --- /dev/null +++ b/docs/changelog/121850.yaml @@ -0,0 +1,5 @@ +pr: 121850 +summary: Take named parameters for identifier and pattern out of snapshot +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/121971.yaml b/docs/changelog/121971.yaml new file mode 100644 index 0000000000000..92d66f81c2e60 --- /dev/null +++ b/docs/changelog/121971.yaml @@ -0,0 +1,5 @@ +pr: 121971 +summary: Do not fetch reserved roles from native store when Get Role API is called +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/122074.yaml b/docs/changelog/122074.yaml new file mode 100644 index 0000000000000..21e171d0eb5e2 --- /dev/null +++ b/docs/changelog/122074.yaml @@ -0,0 +1,8 @@ +pr: 122074 +summary: If the Transform is configured to write to an alias as its destination index, + when the delete_dest_index parameter is set to true, then the Delete API will now + delete the write index backing the alias +area: Transform +type: bug +issues: + - 121913 diff --git a/docs/changelog/122246.yaml b/docs/changelog/122246.yaml new file mode 100644 index 0000000000000..c1e90f3423117 --- /dev/null +++ b/docs/changelog/122246.yaml @@ -0,0 +1,5 @@ +pr: 122246 +summary: Ensure removal of index blocks does not leave key with null value +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/122278.yaml b/docs/changelog/122278.yaml new file mode 100644 index 0000000000000..529d17c729c7f --- /dev/null +++ b/docs/changelog/122278.yaml @@ -0,0 +1,5 @@ +pr: 122278 +summary: Fix serialising the inference update request +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/122293.yaml b/docs/changelog/122293.yaml new file mode 100644 index 0000000000000..31e3da771169e --- /dev/null +++ b/docs/changelog/122293.yaml @@ -0,0 +1,5 @@ +pr: 122293 +summary: Add enterprise license check to inference action for semantic text fields +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/122326.yaml b/docs/changelog/122326.yaml new file mode 100644 index 0000000000000..91c71041d58fc --- /dev/null +++ b/docs/changelog/122326.yaml @@ -0,0 +1,5 @@ +pr: 122326 +summary: System Index Migration Failure Results in a Non-Recoverable State +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/122357.yaml b/docs/changelog/122357.yaml new file mode 100644 index 0000000000000..7648002c9356f --- /dev/null +++ b/docs/changelog/122357.yaml @@ -0,0 +1,6 @@ +pr: 122357 +summary: Handle search timeout in `SuggestPhase` +area: Search +type: bug +issues: + - 122186 diff --git a/docs/changelog/122398.yaml b/docs/changelog/122398.yaml new file mode 100644 index 0000000000000..9096294d810eb --- /dev/null +++ b/docs/changelog/122398.yaml @@ -0,0 +1,6 @@ +pr: 122398 +summary: "[8.18][Inference API] Rename `model_id` prop to model in EIS sparse inference\ + \ request body" +area: Inference +type: enhancement +issues: [] diff --git a/docs/changelog/122427.yaml b/docs/changelog/122427.yaml new file mode 100644 index 0000000000000..2444a0ec894ab --- /dev/null +++ b/docs/changelog/122427.yaml @@ -0,0 +1,5 @@ +pr: 122427 +summary: Improve size limiting string message +area: Infra/Core +type: enhancement +issues: [] diff --git a/docs/changelog/122575.yaml b/docs/changelog/122575.yaml new file mode 100644 index 0000000000000..af72c81b9da8c --- /dev/null +++ b/docs/changelog/122575.yaml @@ -0,0 +1,6 @@ +pr: 122575 +summary: Return an empty suggestion when suggest phase times out +area: Suggesters +type: bug +issues: + - 122548 diff --git a/docs/changelog/122601.yaml b/docs/changelog/122601.yaml new file mode 100644 index 0000000000000..11f44a806917d --- /dev/null +++ b/docs/changelog/122601.yaml @@ -0,0 +1,6 @@ +pr: 122601 +summary: Implicit numeric casting for CASE/GREATEST/LEAST +area: ES|QL +type: bug +issues: + - 121890 diff --git a/docs/changelog/122606.yaml b/docs/changelog/122606.yaml new file mode 100644 index 0000000000000..b6cd40ab6f232 --- /dev/null +++ b/docs/changelog/122606.yaml @@ -0,0 +1,5 @@ +pr: 122606 +summary: Avoid serializing empty `_source` fields in mappings +area: Mapping +type: bug +issues: [] diff --git a/docs/changelog/122653.yaml b/docs/changelog/122653.yaml new file mode 100644 index 0000000000000..e36eb8debf10d --- /dev/null +++ b/docs/changelog/122653.yaml @@ -0,0 +1,6 @@ +pr: 122653 +summary: Knn vector rescoring to sort score docs +area: Vector Search +type: bug +issues: + - 119711 diff --git a/docs/changelog/122731.yaml b/docs/changelog/122731.yaml new file mode 100644 index 0000000000000..afde587a9538b --- /dev/null +++ b/docs/changelog/122731.yaml @@ -0,0 +1,5 @@ +pr: 122731 +summary: Fork post-snapshot-delete cleanup off master thread +area: Snapshot/Restore +type: bug +issues: [] diff --git a/docs/changelog/122762.yaml b/docs/changelog/122762.yaml new file mode 100644 index 0000000000000..d5320e7604f80 --- /dev/null +++ b/docs/changelog/122762.yaml @@ -0,0 +1,6 @@ +pr: 122762 +summary: "ESQL: Remove estimated row size assertion" +area: ES|QL +type: bug +issues: + - 121535 diff --git a/docs/changelog/122905.yaml b/docs/changelog/122905.yaml new file mode 100644 index 0000000000000..eccd50a759734 --- /dev/null +++ b/docs/changelog/122905.yaml @@ -0,0 +1,6 @@ +pr: 122905 +summary: Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks + do not prevent data stream rollover +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/122938.yaml b/docs/changelog/122938.yaml new file mode 100644 index 0000000000000..cfb6e319c6cd2 --- /dev/null +++ b/docs/changelog/122938.yaml @@ -0,0 +1,5 @@ +pr: 122938 +summary: Fix geoip databases index access after system feature migration (again) +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/122951.yaml b/docs/changelog/122951.yaml new file mode 100644 index 0000000000000..b84e05d758fe1 --- /dev/null +++ b/docs/changelog/122951.yaml @@ -0,0 +1,6 @@ +pr: 122951 +summary: Updates the deprecation info API to not warn about system indices and data + streams +area: Indices APIs +type: bug +issues: [] diff --git a/docs/changelog/123076.yaml b/docs/changelog/123076.yaml new file mode 100644 index 0000000000000..270c202f3bbdc --- /dev/null +++ b/docs/changelog/123076.yaml @@ -0,0 +1,5 @@ +pr: 123076 +summary: Retry on streaming errors +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/123155.yaml b/docs/changelog/123155.yaml new file mode 100644 index 0000000000000..73027c87510ba --- /dev/null +++ b/docs/changelog/123155.yaml @@ -0,0 +1,5 @@ +pr: 123155 +summary: Add `ElasticInferenceServiceCompletionServiceSettings` +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/123188.yaml b/docs/changelog/123188.yaml new file mode 100644 index 0000000000000..7e5e5bf7821e7 --- /dev/null +++ b/docs/changelog/123188.yaml @@ -0,0 +1,19 @@ +pr: 123188 +summary: Add deprecation warning to `TransportHandshaker` +area: Infra/Core +type: deprecation +issues: [] +deprecation: + title: Deprecate ability to connect to nodes of versions 8.17 and earlier + area: REST API + details: > + Versions 9.0.0 and later of {es} will not support communication with nodes of versions earlier than 8.18.0, + so the ability to connect to nodes of earlier versions is deprecated in this version. This applies both to + communication within a cluster and communication across clusters (e.g. for <> or + <>). + + {es} will report in its <> each time it opens a connection to a node that + will not be supported from version 9.0.0 onwards. You must upgrade all your clusters to version 8.18.0 or later + before upgrading any of your clusters to 9.0.0 or later. + impact: > + Upgrade all of your clusters to at least 8.18.0 before upgrading any of them to 9.0.0 or later. diff --git a/docs/changelog/123272.yaml b/docs/changelog/123272.yaml new file mode 100644 index 0000000000000..18db2437f85c1 --- /dev/null +++ b/docs/changelog/123272.yaml @@ -0,0 +1,5 @@ +pr: 123272 +summary: Set Connect Timeout to 5s +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/123296.yaml b/docs/changelog/123296.yaml new file mode 100644 index 0000000000000..1dd32d21294fb --- /dev/null +++ b/docs/changelog/123296.yaml @@ -0,0 +1,5 @@ +pr: 123296 +summary: Avoid over collecting in Limit or Lucene Operator +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/123381.yaml b/docs/changelog/123381.yaml new file mode 100644 index 0000000000000..957f11f1b9007 --- /dev/null +++ b/docs/changelog/123381.yaml @@ -0,0 +1,6 @@ +pr: 123381 +summary: Push down `StartsWith` and `EndsWith` functions to Lucene +area: ES|QL +type: enhancement +issues: + - 123067 diff --git a/docs/changelog/123427.yaml b/docs/changelog/123427.yaml new file mode 100644 index 0000000000000..50c29edb79725 --- /dev/null +++ b/docs/changelog/123427.yaml @@ -0,0 +1,5 @@ +pr: 123427 +summary: Reduce iteration complexity for plan traversal +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/123569.yaml b/docs/changelog/123569.yaml new file mode 100644 index 0000000000000..c04601eead9f4 --- /dev/null +++ b/docs/changelog/123569.yaml @@ -0,0 +1,5 @@ +pr: 123569 +summary: Abort pending deletion on `IndicesService` close +area: Store +type: enhancement +issues: [] diff --git a/docs/changelog/123600.yaml b/docs/changelog/123600.yaml new file mode 100644 index 0000000000000..1c7df87d22e4d --- /dev/null +++ b/docs/changelog/123600.yaml @@ -0,0 +1,17 @@ +pr: 123600 +summary: Drop `TLS_RSA` cipher support for JDK 24 +area: TLS +type: breaking +issues: [] +breaking: + title: Drop `TLS_RSA` cipher support for JDK 24 + area: Cluster and node setting + details: >- + This change removes `TLS_RSA` ciphers from the list of default supported ciphers, + for Elasticsearch deployments running on JDK 24. + impact: >- + The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, + `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, + and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. + Please configure your clients to use one of supported cipher suites. + notable: false diff --git a/docs/changelog/123743.yaml b/docs/changelog/123743.yaml new file mode 100644 index 0000000000000..50fccfd6030ae --- /dev/null +++ b/docs/changelog/123743.yaml @@ -0,0 +1,5 @@ +pr: 123743 +summary: Adjust exception thrown when unable to load hunspell dict +area: Analysis +type: bug +issues: [] diff --git a/docs/changelog/123761.yaml b/docs/changelog/123761.yaml new file mode 100644 index 0000000000000..340a235a5bfc1 --- /dev/null +++ b/docs/changelog/123761.yaml @@ -0,0 +1,5 @@ +pr: 123761 +summary: Have create index return a bad request on poor formatting +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/123926.yaml b/docs/changelog/123926.yaml new file mode 100644 index 0000000000000..f93e451f01d6a --- /dev/null +++ b/docs/changelog/123926.yaml @@ -0,0 +1,5 @@ +pr: 123926 +summary: System data streams are not being upgraded in the feature migration API +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/124001.yaml b/docs/changelog/124001.yaml new file mode 100644 index 0000000000000..374a7ad7efb58 --- /dev/null +++ b/docs/changelog/124001.yaml @@ -0,0 +1,7 @@ +pr: 124001 +summary: Use a must boolean statement when pushing down to Lucene when scoring is + also needed +area: ES|QL +type: bug +issues: + - 123967 diff --git a/docs/changelog/124048.yaml b/docs/changelog/124048.yaml new file mode 100644 index 0000000000000..c08fd6f9722ab --- /dev/null +++ b/docs/changelog/124048.yaml @@ -0,0 +1,6 @@ +pr: 124048 +summary: Handle long overflow in dates +area: Search +type: bug +issues: + - 112483 diff --git a/docs/changelog/124149.yaml b/docs/changelog/124149.yaml new file mode 100644 index 0000000000000..07c2f396efda3 --- /dev/null +++ b/docs/changelog/124149.yaml @@ -0,0 +1,5 @@ +pr: 124149 +summary: Retry ILM async action after reindexing data stream +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/124225.yaml b/docs/changelog/124225.yaml new file mode 100644 index 0000000000000..c94c78d83e81f --- /dev/null +++ b/docs/changelog/124225.yaml @@ -0,0 +1,5 @@ +pr: 124225 +summary: "[Inference API] Fix output stream ordering in `InferenceActionProxy`" +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/124266.yaml b/docs/changelog/124266.yaml new file mode 100644 index 0000000000000..3d705bec51474 --- /dev/null +++ b/docs/changelog/124266.yaml @@ -0,0 +1,5 @@ +pr: 124266 +summary: "[8.18] Avoid hoarding cluster state references during rollover" +area: Indices APIs +type: bug +issues: [] diff --git a/docs/changelog/124424.yaml b/docs/changelog/124424.yaml new file mode 100644 index 0000000000000..b514803e22f62 --- /dev/null +++ b/docs/changelog/124424.yaml @@ -0,0 +1,5 @@ +pr: 124424 +summary: Lazy collection copying during node transform +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/124451.yaml b/docs/changelog/124451.yaml new file mode 100644 index 0000000000000..15e3ea2a626ff --- /dev/null +++ b/docs/changelog/124451.yaml @@ -0,0 +1,5 @@ +pr: 124451 +summary: Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. +area: Downsampling +type: bug +issues: [] diff --git a/docs/changelog/124477.yaml b/docs/changelog/124477.yaml new file mode 100644 index 0000000000000..d37a3f27b4dde --- /dev/null +++ b/docs/changelog/124477.yaml @@ -0,0 +1,5 @@ +pr: 124477 +summary: Improve downsample performance by buffering docids and do bulk processing +area: Downsampling +type: enhancement +issues: [] diff --git a/docs/changelog/124527.yaml b/docs/changelog/124527.yaml new file mode 100644 index 0000000000000..5606683cca95a --- /dev/null +++ b/docs/changelog/124527.yaml @@ -0,0 +1,5 @@ +pr: 124527 +summary: Avoid potentially throwing calls to Task#getDescription in model download +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/124597.yaml b/docs/changelog/124597.yaml new file mode 100644 index 0000000000000..1c413bb06d4bf --- /dev/null +++ b/docs/changelog/124597.yaml @@ -0,0 +1,5 @@ +pr: 124597 +summary: Return unique deprecation for old indices with incompatible date formats +area: Infra/Core +type: enhancement +issues: [] diff --git a/docs/changelog/124604.yaml b/docs/changelog/124604.yaml new file mode 100644 index 0000000000000..a13ce104dc97a --- /dev/null +++ b/docs/changelog/124604.yaml @@ -0,0 +1,5 @@ +pr: 124604 +summary: Fix geoip databases index access after system feature migration (take 3) +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/124611.yaml b/docs/changelog/124611.yaml new file mode 100644 index 0000000000000..9db501021cad5 --- /dev/null +++ b/docs/changelog/124611.yaml @@ -0,0 +1,5 @@ +pr: 124611 +summary: Reuse child `outputSet` inside the plan where possible +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/124651.yaml b/docs/changelog/124651.yaml new file mode 100644 index 0000000000000..5c5e064fcd5e8 --- /dev/null +++ b/docs/changelog/124651.yaml @@ -0,0 +1,5 @@ +pr: 124651 +summary: "Fix system data streams to be restorable from a snapshot" +area: Infra/Core +type: bug +issues: [89261] diff --git a/docs/changelog/124670.yaml b/docs/changelog/124670.yaml new file mode 100644 index 0000000000000..202da602d4930 --- /dev/null +++ b/docs/changelog/124670.yaml @@ -0,0 +1,10 @@ +pr: 124670 +summary: Release semantic_text as a GA feature +area: Mapping +type: feature +issues: [] +highlight: + title: Release semantic_text as a GA feature + body: semantic_text is now an official GA (generally available) feature! + This field type allows you to easily set up and perform semantic search with minimal ramp up time. + notable: true diff --git a/docs/changelog/124739.yaml b/docs/changelog/124739.yaml new file mode 100644 index 0000000000000..3c6020231e6fb --- /dev/null +++ b/docs/changelog/124739.yaml @@ -0,0 +1,5 @@ +pr: 124739 +summary: Improve rolling up metrics +area: Downsampling +type: enhancement +issues: [] diff --git a/docs/changelog/124784.yaml b/docs/changelog/124784.yaml new file mode 100644 index 0000000000000..e0b49834f56e1 --- /dev/null +++ b/docs/changelog/124784.yaml @@ -0,0 +1,6 @@ +pr: 124784 +summary: Merge template mappings properly during validation +area: Mapping +type: bug +issues: + - 123372 diff --git a/docs/changelog/124843.yaml b/docs/changelog/124843.yaml new file mode 100644 index 0000000000000..0cad424e5991b --- /dev/null +++ b/docs/changelog/124843.yaml @@ -0,0 +1,5 @@ +pr: 124843 +summary: Ignore _JAVA_OPTIONS +area: Infra/CLI +type: enhancement +issues: [] diff --git a/docs/changelog/124918.yaml b/docs/changelog/124918.yaml new file mode 100644 index 0000000000000..19d3c5a77a3f8 --- /dev/null +++ b/docs/changelog/124918.yaml @@ -0,0 +1,5 @@ +pr: 124918 +summary: Fix EQL double invoking listener +area: EQL +type: bug +issues: [] diff --git a/docs/changelog/124931.yaml b/docs/changelog/124931.yaml new file mode 100644 index 0000000000000..b98595a9a2a0a --- /dev/null +++ b/docs/changelog/124931.yaml @@ -0,0 +1,5 @@ +pr: 124931 +summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. +area: "Snapshot/Restore" +type: bug +issues: [] diff --git a/docs/changelog/125023.yaml b/docs/changelog/125023.yaml new file mode 100644 index 0000000000000..740d2163744c9 --- /dev/null +++ b/docs/changelog/125023.yaml @@ -0,0 +1,5 @@ +pr: 125023 +summary: Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/125073.yaml b/docs/changelog/125073.yaml new file mode 100644 index 0000000000000..a2af9302f4c08 --- /dev/null +++ b/docs/changelog/125073.yaml @@ -0,0 +1,8 @@ +pr: 125073 +summary: "Permanently switch from Java SecurityManager to Entitlements. + The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order + to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. + Starting with this version, Entitlements will permanently replace the Java SecurityManager." +area: Infra/Core +type: upgrade +issues: [] diff --git a/docs/changelog/125103.yaml b/docs/changelog/125103.yaml new file mode 100644 index 0000000000000..da5d025e77869 --- /dev/null +++ b/docs/changelog/125103.yaml @@ -0,0 +1,5 @@ +pr: 125103 +summary: Fix LTR query feature with phrases (and two-phase) queries +area: Ranking +type: bug +issues: [] diff --git a/docs/changelog/125159.yaml b/docs/changelog/125159.yaml new file mode 100644 index 0000000000000..ccef5cbd4cbc5 --- /dev/null +++ b/docs/changelog/125159.yaml @@ -0,0 +1,5 @@ +pr: 125159 +summary: Update bundled JDK to Java 24 +area: Packaging +type: upgrade +issues: [] diff --git a/docs/changelog/125171.yaml b/docs/changelog/125171.yaml new file mode 100644 index 0000000000000..717e23d8d67b8 --- /dev/null +++ b/docs/changelog/125171.yaml @@ -0,0 +1,5 @@ +pr: 125171 +summary: Reindex data stream indices on different nodes +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/125345.yaml b/docs/changelog/125345.yaml new file mode 100644 index 0000000000000..b3137739b70fc --- /dev/null +++ b/docs/changelog/125345.yaml @@ -0,0 +1,6 @@ +pr: 125345 +summary: ESQL - date nanos range bug? +area: ES|QL +type: bug +issues: + - 125439 diff --git a/docs/changelog/125352.yaml b/docs/changelog/125352.yaml new file mode 100644 index 0000000000000..0895732b0b5bf --- /dev/null +++ b/docs/changelog/125352.yaml @@ -0,0 +1,5 @@ +pr: 125352 +summary: Fix NPE in rolling over unknown target and return 404 +area: Indices APIs +type: bug +issues: [] diff --git a/docs/changelog/125370.yaml b/docs/changelog/125370.yaml new file mode 100644 index 0000000000000..113988089776c --- /dev/null +++ b/docs/changelog/125370.yaml @@ -0,0 +1,6 @@ +pr: 125370 +summary: Set default similarity for Cohere model to cosine +area: Machine Learning +type: bug +issues: + - 122878 diff --git a/docs/changelog/125389.yaml b/docs/changelog/125389.yaml new file mode 100644 index 0000000000000..a57524ff40db5 --- /dev/null +++ b/docs/changelog/125389.yaml @@ -0,0 +1,6 @@ +pr: 125389 +summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs + as archive in current version. +area: Search +type: bug +issues: [] diff --git a/docs/changelog/125404.yaml b/docs/changelog/125404.yaml new file mode 100644 index 0000000000000..c9dd47b3f3263 --- /dev/null +++ b/docs/changelog/125404.yaml @@ -0,0 +1,5 @@ +pr: 125404 +summary: Check if the anomaly results index has been rolled over +area: Machine Learning +type: upgrade +issues: [] diff --git a/docs/changelog/125595.yaml b/docs/changelog/125595.yaml new file mode 100644 index 0000000000000..2a3f4c61c07b0 --- /dev/null +++ b/docs/changelog/125595.yaml @@ -0,0 +1,5 @@ +pr: 125595 +summary: Esql - Fix lucene push down behavior when a range contains nanos and millis +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/125636.yaml b/docs/changelog/125636.yaml new file mode 100644 index 0000000000000..3ceaade9f01c2 --- /dev/null +++ b/docs/changelog/125636.yaml @@ -0,0 +1,6 @@ +pr: 125636 +summary: Make `numberOfChannels` consistent with layout map by removing duplicated + `ChannelSet` +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/125650.yaml b/docs/changelog/125650.yaml new file mode 100644 index 0000000000000..89bacc3cee4a1 --- /dev/null +++ b/docs/changelog/125650.yaml @@ -0,0 +1,7 @@ +pr: 125650 +summary: Load `FieldInfos` from store if not yet initialised through a refresh on + `IndexShard` +area: Search +type: bug +issues: + - 125483 diff --git a/docs/changelog/125659.yaml b/docs/changelog/125659.yaml new file mode 100644 index 0000000000000..d0f2266e94562 --- /dev/null +++ b/docs/changelog/125659.yaml @@ -0,0 +1,6 @@ +pr: 125659 +summary: Non existing synonyms sets do not fail shard recovery for indices +area: "Analysis" +type: bug +issues: + - 125603 diff --git a/docs/changelog/125716.yaml b/docs/changelog/125716.yaml new file mode 100644 index 0000000000000..fa85812c72c1a --- /dev/null +++ b/docs/changelog/125716.yaml @@ -0,0 +1,5 @@ +pr: 125716 +summary: Return appropriate error on null dims update instead of npe +area: Vector Search +type: bug +issues: [] diff --git a/docs/changelog/125732.yaml b/docs/changelog/125732.yaml new file mode 100644 index 0000000000000..e375921ade280 --- /dev/null +++ b/docs/changelog/125732.yaml @@ -0,0 +1,5 @@ +pr: 125732 +summary: Log stack traces on data nodes before they are cleared for transport +area: Search +type: bug +issues: [] diff --git a/docs/changelog/125764.yaml b/docs/changelog/125764.yaml new file mode 100644 index 0000000000000..8f85645c1425c --- /dev/null +++ b/docs/changelog/125764.yaml @@ -0,0 +1,8 @@ +pr: 125764 +summary: Fix `ReplaceMissingFieldsWithNull` +area: ES|QL +type: bug +issues: + - 126036 + - 121754 + - 126030 diff --git a/docs/changelog/125916.yaml b/docs/changelog/125916.yaml new file mode 100644 index 0000000000000..57741e4f0870a --- /dev/null +++ b/docs/changelog/125916.yaml @@ -0,0 +1,5 @@ +pr: 125916 +summary: Re-enable parallel collection for field sorted top hits +area: Search +type: bug +issues: [] diff --git a/docs/changelog/126077.yaml b/docs/changelog/126077.yaml new file mode 100644 index 0000000000000..ffb98d71ba630 --- /dev/null +++ b/docs/changelog/126077.yaml @@ -0,0 +1,6 @@ +pr: 126077 +summary: Preventing `ConcurrentModificationException` when updating settings for more + than one index +area: Indices APIs +type: bug +issues: [] diff --git a/docs/changelog/126191.yaml b/docs/changelog/126191.yaml new file mode 100644 index 0000000000000..b5ab684e057a6 --- /dev/null +++ b/docs/changelog/126191.yaml @@ -0,0 +1,5 @@ +pr: 126191 +summary: Fix NPE for missing Content Type header in OIDC Authenticator +area: Authentication +type: bug +issues: [] diff --git a/docs/changelog/126273.yaml b/docs/changelog/126273.yaml new file mode 100644 index 0000000000000..420c0eb317a03 --- /dev/null +++ b/docs/changelog/126273.yaml @@ -0,0 +1,5 @@ +pr: 126273 +summary: Fix LTR rescorer with model alias +area: Ranking +type: bug +issues: [] diff --git a/docs/changelog/126376.yaml b/docs/changelog/126376.yaml new file mode 100644 index 0000000000000..5ac6bc747f160 --- /dev/null +++ b/docs/changelog/126376.yaml @@ -0,0 +1,6 @@ +pr: 126376 +summary: Set `keyUsage` for generated HTTP certificates and self-signed CA +area: TLS +type: bug +issues: + - 117769 diff --git a/docs/changelog/126411.yaml b/docs/changelog/126411.yaml new file mode 100644 index 0000000000000..2455fbbdb9e88 --- /dev/null +++ b/docs/changelog/126411.yaml @@ -0,0 +1,6 @@ +pr: 126411 +summary: Fix usage of already released null block in `ValueSourceReaderOperator` +area: ES|QL +type: bug +issues: + - 125850 diff --git a/docs/changelog/126637.yaml b/docs/changelog/126637.yaml new file mode 100644 index 0000000000000..6b51566457bfc --- /dev/null +++ b/docs/changelog/126637.yaml @@ -0,0 +1,5 @@ +pr: 126637 +summary: Improve resiliency of `UpdateTimeSeriesRangeService` +area: TSDB +type: bug +issues: [] diff --git a/docs/changelog/126686.yaml b/docs/changelog/126686.yaml new file mode 100644 index 0000000000000..802ec538e5c1e --- /dev/null +++ b/docs/changelog/126686.yaml @@ -0,0 +1,6 @@ +pr: 126686 +summary: Fix race condition in `RestCancellableNodeClient` +area: Task Management +type: bug +issues: + - 88201 diff --git a/docs/internal/Versioning.md b/docs/internal/Versioning.md index f0f730f618259..474278e873922 100644 --- a/docs/internal/Versioning.md +++ b/docs/internal/Versioning.md @@ -35,19 +35,19 @@ Every change to the transport protocol is represented by a new transport version higher than all previous transport versions, which then becomes the highest version recognized by that build of Elasticsearch. The version ids are stored as constants in the `TransportVersions` class. -Each id has a standard pattern `M_NNN_SS_P`, where: +Each id has a standard pattern `M_NNN_S_PP`, where: * `M` is the major version * `NNN` is an incrementing id -* `SS` is used in subsidiary repos amending the default transport protocol -* `P` is used for patches and backports +* `S` is used in subsidiary repos amending the default transport protocol +* `PP` is used for patches and backports When you make a change to the serialization form of any object, you need to create a new sequential constant in `TransportVersions`, introduced in the same PR that adds the change, that increments the `NNN` component from the previous highest version, with other components set to zero. -For example, if the previous version number is `8_413_00_1`, -the next version number should be `8_414_00_0`. +For example, if the previous version number is `8_413_0_01`, +the next version number should be `8_414_0_00`. Once you have defined your constant, you then need to use it in serialization code. If the transport version is at or above the new id, @@ -166,7 +166,7 @@ also has that change, and knows about the patch backport ids and what they mean. Index version is a single incrementing version number for the index data format, metadata, and associated mappings. It is declared the same way as the -transport version - with the pattern `M_NNN_SS_P`, for the major version, version id, +transport version - with the pattern `M_NNN_S_PP`, for the major version, version id, subsidiary version id, and patch number respectively. Index version is stored in index metadata when an index is created, diff --git a/docs/plugins/development/creating-classic-plugins.asciidoc b/docs/plugins/development/creating-classic-plugins.asciidoc index 58dc00e496c2d..638af4a1c3d0f 100644 --- a/docs/plugins/development/creating-classic-plugins.asciidoc +++ b/docs/plugins/development/creating-classic-plugins.asciidoc @@ -49,42 +49,140 @@ to install your plugin for testing. The Java plugin is auto-loaded only if it's `plugins/` directory. [discrete] -[[plugin-authors-jsm]] -==== Java Security permissions - -Some plugins may need additional security permissions. A plugin can include -the optional `plugin-security.policy` file containing `grant` statements for -additional permissions. Any additional permissions will be displayed to the user -with a large warning, and they will have to confirm them when installing the -plugin interactively. So if possible, it is best to avoid requesting any -spurious permissions! - -If you are using the {es} Gradle build system, place this file in -`src/main/plugin-metadata` and it will be applied during unit tests as well. - -The Java security model is stack-based, and additional -permissions are granted to the jars in your plugin, so you have to -write proper security code around operations requiring elevated privileges. -You might add a check to prevent unprivileged code (such as scripts) -from gaining escalated permissions. For example: - -[source,java] --------------------------------------------------- -// ES permission you should check before doPrivileged() blocks -import org.elasticsearch.SpecialPermission; - -SecurityManager sm = System.getSecurityManager(); -if (sm != null) { - // unprivileged code such as scripts do not have SpecialPermission - sm.checkPermission(new SpecialPermission()); -} -AccessController.doPrivileged( - // sensitive operation -); --------------------------------------------------- - -Check https://www.oracle.com/technetwork/java/seccodeguide-139067.html[Secure Coding Guidelines for Java SE] -for more information. +[[_entitlements_policy]] +==== Entitlements policy + +Some plugins may need additional _entitlements_. + +{es} limits the ability to perform certain security-sensitive actions as part of its _Entitlement_ security mechanism (e.g. to limit the potential fallout from remote code execution (RCE) vulnerabilities). + +The Entitlement model is based on Java modules. +An _entitlement_ granted to a Java module allows the module's code to perform the security-sensitive action associated with that entitlement. For example, the ability to create threads is limited to modules that have the `manage_threads` entitlement; likewise, the ability to read a file from the filesystem is limited to modules that have the `files` entitlement for that particular file. + +In practice, an entitlement allows plugin code to call a well-defined set of corresponding JDK methods; without the entitlement calls to those JDK methods are denied and throw a `NotEntitledException`. Plugin can include the optional `entitlement-policy.yaml` file to define modules and required entitlements. Any additional entitlement requested by the plugin will be displayed to the user with a large warning, and users will have to confirm them when installing the plugin interactively. Therefore, it is best to avoid requesting any spurious entitlement! + +If you are using the {es} Gradle build system, place this file in `src/main/plugin-metadata` and it will be applied during unit tests as well. + +An entitlement policy applies to all of your plugin jars (your own code and third party dependencies). You have to write your policy file accordingly. For example, if a plugin uses the Example API client to perform network operations, it will need a policy that may look like this: + +```YAML +org.elasticsearch.example-plugin: + - manage_threads +com.example.api.client: + - set_https_connection_properties + - outbound_network +``` + +Note how the network related entitlements are granted to the `com.example.api.client` module, as the code performing the sensitive network operations is in the `example-api-client` dependency. + +If your plugin is not modular, all entitlements must be specified under the catch-all `ALL-UNNAMED` module name: + +```YAML +ALL-UNNAMED: + - manage_threads + - set_https_connection_properties + - outbound_network +``` +==== Entitlements + +The entitlements currently implemented and enforced in {es} that are available to plugins are the following ones: + +===== `manage_threads` + +Allows code to call methods that create or modify properties on Java Threads, for example `Thread#start` or `ThreadGroup#setMaxPriority`. In general, setting the name, priority, daemon state and context class loader are things no plugins should do when executing on +{es} threadpools; however, many 3rd party libraries that support async operations (e.g. Apache HTTP client) need to manage their own threads. In this case it is justifiable to request this entitlement. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - manage_threads +``` + +===== `outbound_network` + +Allows code to call methods to make a network connection. {es} does not grant any network access by default; each plugin that needs to directly connect to an external resource (e.g. to upload or download data) must request this entitlement. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - outbound_network +``` + +===== `set_https_connection_properties` +Allows code to call methods to change properties on an established HTTPS connection. While this is generally innocuous (e.g. the google API client uses it to modify the HTTPS connections they just created), these methods can allow code to change arbitrary connections. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - set_https_connection_properties +``` + +===== `inbound_network` (deprecated) +Allows code to call methods to listen for incoming connections, so external resources can connect directly to your plugin. This entitlement should only be used when absolutely necessary (e.g. if a library you depend on requires it for authentication). Granting it makes the {es} node more vulnerable to attacks. This entitlement is deprecated, and can be removed in a future version of {es}. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - inbound_network +``` + +===== `load_native_libraries` +Allows code to load native libraries and call https://docs.oracle.com/en/java/javase/24/core/restricted-methods.html[restricted methods]. This entitlement also enables native access for modules it is granted to. Native code may alter the JVM or circumvent access checks such as file or network restrictions. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - load_native_libraries +``` + +===== `files` + +Allows code to access the filesystem, to read or write paths as specified by the entitlement's fields. The filesystem of the OS hosting {es} may contain sensitive files, for example credentials. Some files are meant to be always accessible to {es}, but plugins can not access them directly: {es} enforces that certain files can only be read by its core code, while some other files can not be read or written at all. A plugin is always granted `read` access to the {es} config directory and `read_write` access to the temp directory; if the plugin requires to read, write or access additional files or directories, it must specify them via this entitlement. + +It is possible to specify 3 different types of file entitlement: + +* `path` to specify an absolute path +* `relative_path` to specify a relative path. The path will be resolved via the `relative_to` field, which is used to qualify the relative path. It can be a specific {es} directory (`config` or `data`), or to the user home directory (`home`) (the home of the user running {es}) +* `relative_path` to specify a path resolved via the `relative_to` field, which can have the following values: + - `config`: the {es} https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html#config-files-location[config directory] + - `data`: the {es} https://www.elastic.co/guide/en/elasticsearch/reference/current/path-settings-overview.html[data directory] + - `home`: the home directory of the user running {es} + - `path_setting` to specify a path defined via an {es} setting. The path can be absolute or relative; in the latter case, the path will be resolved using the `basedir_if_relative` path (which can assume the same values as `relative_to`) + +Each of the 3 types has some additional fields: + +* `mode` (required): can be either `read` or `read_write` +* `platform` (optional): indicates this item applies only to one platform, which can be one of `linux`, `macos` or `windows`. On other platforms, the item is ignored. If this field is not specified, the item applies to all platforms. +* `exclusive`: access to this path is exclusive for this plugin; this means that other plugins will not be able to access to it, not even if they have an entitlement that would normally grant access to that path. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - files: + - path: "/absolute/path" + mode: read + - relative_path: "relative/file.txt" + relative_to: data + mode: read_write + - path_setting: setting.name + basedir_if_relative: data + mode: read +``` + + +===== `write_system_properties` +Allows code to set one or more system properties (e.g. by calling `System#setProperty`). The code to which this entitlement is granted can change the properties listed in the `properties` field. In general, it's best to avoid changing a system property dynamically as this can have effects on code which later reads the property. The global nature of system properties means one plugin could then affect another, depending on load order. + +Example: +```yaml +org.example.module: # or 'ALL-UNNAMED' if the plugin is non-modular + - write_system_properties: + properties: + - property.one + - property.two +``` +Check the Entitlements {es-repo}tree/main/libs/entitlement/README.md[README in the elasticsearch repository] for more information. + [[plugin-descriptor-file-classic]] ==== The plugin descriptor file for classic plugins diff --git a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc index b719ea376a279..457e7e544ad8e 100644 --- a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc @@ -32,7 +32,7 @@ To see how the `flatten_graph` filter works, you first need to produce a token graph containing multi-position tokens. The following <> request uses the `synonym_graph` -filter to add `dns` as a multi-position synonym for `domain name system` in the +filter to add `internet phonebook` as a multi-position synonym for `domain name system` in the text `domain name system is fragile`: [source,console] @@ -43,17 +43,17 @@ GET /_analyze "filter": [ { "type": "synonym_graph", - "synonyms": [ "dns, domain name system" ] + "synonyms": [ "internet phonebook, domain name system" ] } ], "text": "domain name system is fragile" } ---- -The filter produces the following token graph with `dns` as a multi-position +The filter produces the following token graph with `internet phonebook` as a multi-position token. -image::images/analysis/token-graph-dns-synonym-ex.svg[align="center"] +image::images/analysis/token-graph-dns-synonym-ex2.svg[align="center"] //// [source,console-result] @@ -61,47 +61,55 @@ image::images/analysis/token-graph-dns-synonym-ex.svg[align="center"] { "tokens": [ { - "token": "dns", + "token": "internet", "start_offset": 0, "end_offset": 18, "type": "SYNONYM", - "position": 0, - "positionLength": 3 + "position": 0 }, { "token": "domain", "start_offset": 0, "end_offset": 6, "type": "", - "position": 0 + "position": 0, + "positionLength": 2 + }, + { + "token": "phonebook", + "start_offset": 0, + "end_offset": 18, + "type": "SYNONYM", + "position": 1, + "positionLength": 3 }, { "token": "name", "start_offset": 7, "end_offset": 11, "type": "", - "position": 1 + "position": 2 }, { "token": "system", "start_offset": 12, "end_offset": 18, "type": "", - "position": 2 + "position": 3 }, { "token": "is", "start_offset": 19, "end_offset": 21, "type": "", - "position": 3 + "position": 4 }, { "token": "fragile", "start_offset": 22, "end_offset": 29, "type": "", - "position": 4 + "position": 5 } ] } @@ -122,7 +130,7 @@ GET /_analyze "filter": [ { "type": "synonym_graph", - "synonyms": [ "dns, domain name system" ] + "synonyms": [ "internet phonebook, domain name system" ] }, "flatten_graph" ], @@ -133,7 +141,7 @@ GET /_analyze The filter produces the following flattened token graph, which is suitable for indexing. -image::images/analysis/token-graph-dns-invalid-ex.svg[align="center"] +image::images/analysis/token-graph-dns-synonym-flattened-ex2.svg[align="center"] //// [source,console-result] @@ -141,12 +149,11 @@ image::images/analysis/token-graph-dns-invalid-ex.svg[align="center"] { "tokens": [ { - "token": "dns", + "token": "internet", "start_offset": 0, - "end_offset": 18, + "end_offset": 6, "type": "SYNONYM", - "position": 0, - "positionLength": 3 + "position": 0 }, { "token": "domain", @@ -155,6 +162,14 @@ image::images/analysis/token-graph-dns-invalid-ex.svg[align="center"] "type": "", "position": 0 }, + { + "token": "phonebook", + "start_offset": 7, + "end_offset": 18, + "type": "SYNONYM", + "position": 1, + "positionLength": 2 + }, { "token": "name", "start_offset": 7, @@ -224,4 +239,4 @@ PUT /my-index-000001 } } } ----- \ No newline at end of file +---- diff --git a/docs/reference/commands/certutil.asciidoc b/docs/reference/commands/certutil.asciidoc index 6720aef470049..9f7588b61f1c6 100644 --- a/docs/reference/commands/certutil.asciidoc +++ b/docs/reference/commands/certutil.asciidoc @@ -11,7 +11,7 @@ use with Transport Layer Security (TLS) in the {stack}. -------------------------------------------------- bin/elasticsearch-certutil ( -(ca [--ca-dn ] [--days ] [--pem]) +(ca [--ca-dn ] [--keyusage ] [--days ] [--pem]) | (cert ([--ca ] | [--ca-cert --ca-key ]) [--ca-dn ] [--ca-pass ] [--days ] @@ -158,6 +158,10 @@ parameter is only applicable to the `cert` parameter. `--ca-pass `:: Specifies the password for an existing CA private key or the generated CA private key. This parameter is only applicable to the `cert` parameter +`--keyusage `:: Specifies a comma-separated list of key usage restrictions +(as per RFC 5280) that are used for the generated CA certificate. The default value +is `keyCertSign,cRLSign`. This parameter may only be used with the `ca` parameter. + `--days `:: Specifies an integer value that represents the number of days the generated certificates are valid. The default value is `1095`. This parameter cannot be used with the `csr` or `http` parameters. diff --git a/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc b/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc index db536099f1aad..7d3eebae387e0 100644 --- a/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc +++ b/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc @@ -59,7 +59,7 @@ docker run \ --network "elastic" \ --tty \ --rm \ -docker.elastic.co/enterprise-search/elastic-connectors:{version}.0 \ +docker.elastic.co/enterprise-search/elastic-connectors:{version} \ /app/bin/elastic-ingest \ -c /config/config.yml ---- @@ -73,4 +73,4 @@ Find all available Docker images in the https://www.docker.elastic.co/r/enterpri ==== We also have a quickstart self-managed option using Docker Compose, so you can spin up all required services at once: Elasticsearch, Kibana, and the connectors service. Refer to this https://github.com/elastic/connectors/tree/main/scripts/stack#readme[README] in the `elastic/connectors` repo for more information. -==== \ No newline at end of file +==== diff --git a/docs/reference/connector/docs/connectors-API-tutorial.asciidoc b/docs/reference/connector/docs/connectors-API-tutorial.asciidoc index 2e26e0d2a361d..8e9c7de92128d 100644 --- a/docs/reference/connector/docs/connectors-API-tutorial.asciidoc +++ b/docs/reference/connector/docs/connectors-API-tutorial.asciidoc @@ -84,7 +84,7 @@ Note: With {es} running locally, you will need to pass the username and password .Running API calls **** -You can run API calls using the https://www.elastic.co/guide/en/kibana/master/console-kibana.html[Dev Tools Console] in Kibana, using `curl` in your terminal, or with our programming language clients. +You can run API calls using the https://www.elastic.co/guide/en/kibana/8.x/console-kibana.html[Dev Tools Console] in Kibana, using `curl` in your terminal, or with our programming language clients. Our example widget allows you to copy code examples in both Dev Tools Console syntax and curl syntax. To use curl, you'll need to add authentication headers to your request. @@ -171,9 +171,9 @@ Now it's time for the real fun! We'll set up a connector to create a searchable [discrete#es-connectors-tutorial-api-create-connector] ==== Create a connector -We'll use the https://www.elastic.co/guide/en/elasticsearch/reference/master/create-connector-api.html[Create connector API] to create a PostgreSQL connector instance. +We'll use the https://www.elastic.co/guide/en/elasticsearch/reference/8.x/create-connector-api.html[Create connector API] to create a PostgreSQL connector instance. -Run the following API call, using the https://www.elastic.co/guide/en/kibana/master/console-kibana.html[Dev Tools Console] or `curl`: +Run the following API call, using the https://www.elastic.co/guide/en/kibana/8.x/console-kibana.html[Dev Tools Console] or `curl`: [source,console] ---- diff --git a/docs/reference/connector/docs/connectors-release-notes.asciidoc b/docs/reference/connector/docs/connectors-release-notes.asciidoc index ff3d859e1a888..fbf612619c076 100644 --- a/docs/reference/connector/docs/connectors-release-notes.asciidoc +++ b/docs/reference/connector/docs/connectors-release-notes.asciidoc @@ -9,6 +9,18 @@ Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation]. ==== +[[es-connectors-release-notes-8-18-0]] +=== 8.18.0 + +[discrete] +[[es-connectors-release-notes-8-18-0-enhancements]] +==== Enhancements + +* Updated the OSS Dockerfile in the https://github.com/elastic/connectors[Connectors] repository to use a different base image for improved security. ++ +IMPORTANT: Users building custom docker images based on this Dockerfile may have to review their configuration for compatibility with the new base image. +See https://github.com/elastic/connectors/pull/3063[*PR 3063*]. + [discrete] [[es-connectors-release-notes-8-17-0]] === 8.17.0 diff --git a/docs/reference/connector/docs/connectors-run-from-docker.asciidoc b/docs/reference/connector/docs/connectors-run-from-docker.asciidoc index 1688b945f05bc..8c4b438a87f8c 100644 --- a/docs/reference/connector/docs/connectors-run-from-docker.asciidoc +++ b/docs/reference/connector/docs/connectors-run-from-docker.asciidoc @@ -62,7 +62,7 @@ docker run \ --rm \ --tty -i \ --network host \ -docker.elastic.co/enterprise-search/elastic-connectors:{version}.0 \ +docker.elastic.co/enterprise-search/elastic-connectors:{version} \ /app/bin/elastic-ingest \ -c /config/config.yml ---- @@ -70,7 +70,7 @@ docker.elastic.co/enterprise-search/elastic-connectors:{version}.0 \ [TIP] ==== For unreleased versions, append the `-SNAPSHOT` suffix to the version number. -For example, `docker.elastic.co/enterprise-search/elastic-connectors:8.14.0.0-SNAPSHOT`. +For example, `docker.elastic.co/enterprise-search/elastic-connectors:8.17.0-SNAPSHOT`. ==== Find all available Docker images in the https://www.docker.elastic.co/r/enterprise-search/elastic-connectors[official registry]. diff --git a/docs/reference/connector/docs/connectors-servicenow.asciidoc b/docs/reference/connector/docs/connectors-servicenow.asciidoc index 3dc98ed9a44c9..e97d2b20399c7 100644 --- a/docs/reference/connector/docs/connectors-servicenow.asciidoc +++ b/docs/reference/connector/docs/connectors-servicenow.asciidoc @@ -144,6 +144,13 @@ For default services, connectors use the following roles to find users who have For services other than these defaults, the connector iterates over access controls with `read` operations and finds the respective roles for those services. +[IMPORTANT] +==== +The ServiceNow connector applies access control at the service (table) level. +This means documents within a given ServiceNow table share the same access control settings. +Users with permission to a table can access all documents from that table in Elasticsearch. +==== + [NOTE] ==== The ServiceNow connector does not support scripted and conditional permissions. @@ -379,6 +386,13 @@ For default services, connectors use the following roles to find users who have For services other than these defaults, the connector iterates over access controls with `read` operations and finds the respective roles for those services. +[IMPORTANT] +==== +The ServiceNow connector applies access control at the service (table) level. +This means documents within a given ServiceNow table share the same access control settings. +Users with permission to a table can access all documents from that table in Elasticsearch. +==== + [NOTE] ==== The ServiceNow connector does not support scripted and conditional permissions. diff --git a/docs/reference/connector/docs/index.asciidoc b/docs/reference/connector/docs/index.asciidoc index dfca45f86ebce..3a398b0c38e64 100644 --- a/docs/reference/connector/docs/index.asciidoc +++ b/docs/reference/connector/docs/index.asciidoc @@ -62,7 +62,7 @@ include::_connectors-list-clients.asciidoc[] == Connector framework All Elastic connectors are built using our Python connector framework. -The source code is available in the {connectors-python}[`elastic/connectors`] repository on GitHub. +The source code is available in the https://github.com/elastic/connectors[`elastic/connectors`] repository on GitHub. The connector framework is available for developers to customize existing self-managed connectors or build their own connectors. Refer to <> for details. diff --git a/docs/reference/data-streams/logs.asciidoc b/docs/reference/data-streams/logs.asciidoc index 7058cfe51496f..3f2130161bcfb 100644 --- a/docs/reference/data-streams/logs.asciidoc +++ b/docs/reference/data-streams/logs.asciidoc @@ -14,13 +14,15 @@ stream. The exact impact varies by data set. [[how-to-use-logsds]] === Create a logs data stream +IMPORTANT: Fleet integrations use <> managed by Elastic. To modify these backing templates, update their {observability-guide}/logs-index-template.html#custom-logs-template-edit[composite `custom` templates]. + To create a logs data stream, set your <> `index.mode` to `logsdb`: [source,console] ---- PUT _index_template/my-index-template { - "index_patterns": ["logs-*"], + "index_patterns": ["my-datastream-*"], "data_stream": { }, "template": { "settings": { @@ -112,7 +114,9 @@ IMPORTANT: On existing data streams, `logsdb` mode is applied on <> -* <> * All <> Accepted metric types vary based on the field type: diff --git a/docs/reference/esql/esql-commands.asciidoc b/docs/reference/esql/esql-commands.asciidoc index 33e748d7eb7c1..e200ce760f110 100644 --- a/docs/reference/esql/esql-commands.asciidoc +++ b/docs/reference/esql/esql-commands.asciidoc @@ -42,9 +42,7 @@ ifeval::["{release-state}"=="unreleased"] endif::[] * <> * <> -ifeval::["{release-state}"=="unreleased"] -//* experimental:[] <> -endif::[] +* experimental:[] <> * experimental:[] <> * <> * <> @@ -67,9 +65,7 @@ ifeval::["{release-state}"=="unreleased"] endif::[] include::processing-commands/keep.asciidoc[] include::processing-commands/limit.asciidoc[] -ifeval::["{release-state}"=="unreleased"] -//include::processing-commands/lookup.asciidoc[] -endif::[] +include::processing-commands/lookup.asciidoc[] include::processing-commands/mv_expand.asciidoc[] include::processing-commands/rename.asciidoc[] include::processing-commands/sort.asciidoc[] diff --git a/docs/reference/esql/esql-enrich-data.asciidoc b/docs/reference/esql/esql-enrich-data.asciidoc index ad34e29f1a55b..34125fb74d32c 100644 --- a/docs/reference/esql/esql-enrich-data.asciidoc +++ b/docs/reference/esql/esql-enrich-data.asciidoc @@ -15,6 +15,15 @@ For example, you can use `ENRICH` to: * Add product information to retail orders based on product IDs * Supplement contact information based on an email address +<> is similar to <> in the fact that they both help you join data together. You should use `ENRICH` when: + +* Enrichment data doesn't change frequently +* You can accept index-time overhead +* You can accept having multiple matches combined into multi-values +* You can accept being limited to predefined match fields +* You do not need fine-grained security: There are no restrictions to specific enrich policies or document and field level security. +* You want to match using ranges or spatial relations + [discrete] [[esql-how-enrich-works]] ==== How the `ENRICH` command works diff --git a/docs/reference/esql/esql-for-search.asciidoc b/docs/reference/esql/esql-for-search.asciidoc new file mode 100644 index 0000000000000..7254c06a86ff0 --- /dev/null +++ b/docs/reference/esql/esql-for-search.asciidoc @@ -0,0 +1,215 @@ +[[esql-for-search]] +=== Using {esql} for search + +preview::[] + +This page provides an overview of how to use {esql} for search use cases. + +[TIP] +==== +Prefer to get started with a hands-on tutorial? Check out <>. +==== + +The following table summarizes the key search features available in {esql} and when they were introduced. + +[cols="1,1,2", options="header"] +|=== +|Feature |Available since |Description + +|<> +|8.17 +|Perform basic text searches with <> and <> + +|<> +|8.17 +|Execute complex queries with <> using Query String syntax + +|<> +|8.18/9.0 +|Calculate and sort by relevance with `METADATA _score` + +|Enhanced match options +|8.18/9.0 +|Configure text searches with additional parameters for the `match` function + +|<> +|8.18/9.0 +|Use Kibana Query Language with <> function + +|<> +|8.18/9.0 +|Perform semantic searches on `semantic_text` field types + +|<> +|8.18/9.0 +|Combine lexical and semantic search approaches with custom weights +|=== + +[[esql-filtering-vs-searching]] +==== Filtering vs. searching + +{esql} can be used for both simple filtering and relevance-based searching: + +* **Filtering** removes non-matching documents without calculating relevance scores +* **Searching** both filters documents and ranks them by how well they match the query + +Note that filtering is faster than searching, because it doesn't require score calculations. + +[[esql-for-search-scoring]] +===== Relevance scoring + +To get the most relevant results first, you need to use `METADATA _score` and sort by score. For example: + +[source,esql] +---- +FROM books METADATA _score +| WHERE match(title, "Shakespeare") OR match(plot, "Shakespeare") +| SORT _score DESC +---- + +[[esql-for-search-how-scoring-works]] +===== How `_score` works + +When working with relevance scoring in ES|QL: + +* If you don't include `METADATA _score` in your query, this only performs filtering operations with no relevance calculation. +* When you include `METADATA _score`, any search function included in `WHERE` conditions contribute to the relevance score. This means that every occurrence of `MATCH`, `QSTR` and `KQL` will affect the score. +* Filtering operations that are not search functions, like range conditions and exact matches, don't affect the score. +* Including `METADATA _score` doesn't automatically sort your results by relevance. You must explicitly use `SORT _score DESC` or `SORT _score ASC` to order your results by relevance. + +[[esql-for-search-full-text]] +==== Full text search + +[[esql-for-search-match-function-operator]] +===== Match function and operator + +ES|QL offers two syntax options for `match`, which replicate the functionality of <> queries in Query DSL. + +Use the compact operator syntax (`:`) for simple text matching with default parameters. + +[source,esql] +---- +FROM logs | WHERE match(message, "connection error") +---- + +Use the `match()` function syntax when you need to pass additional parameters: + +[source,esql] +---- +FROM products | WHERE match(name, "laptop", { "boost": 2.0 }) +---- + +These full-text functions address several key limitations that existed for text filtering in {esql}: + +* They work directly on multivalued fields, returning results when any value in a multivalued field matches the query +* They leverage analyzers, ensuring the query is analyzed with the same process as the indexed data (enabling case-insensitive matching, ASCII folding, stopword removal, and synonym support) +* They are highly performant, using Lucene index structures rather than pattern matching or regular expressions to locate terms in your data + +Refer to this blog for more context: https://www.elastic.co/search-labs/blog/filtering-in-esql-full-text-search-match-qstr[Introducing full text filtering in ES|QL]. + +[TIP] +==== +See <> for more advanced options using match. +==== + +[IMPORTANT] +==== +These queries match documents but don't automatically sort by relevance. To get the most relevant results first, you need to use `METADATA _score` and sort by score. See <> for more information. +==== + +[[esql-for-search-query-string]] +===== Query string function (`QSTR`) + +The <> provides the same functionality as the Query DSL's `query_string` query. This is for advanced use cases, such as wildcard searches, searches across multiple fields, and more. + +[source,esql] +---- +FROM articles METADATA _score +| WHERE QSTR("(new york city) OR (big apple)") +| SORT _score DESC +| LIMIT 10 +---- + +For complete details, refer to the <>. + +[[esql-for-search-kql]] +===== Kibana Query Language function (`KQL`) + +Use the <> to use the {kibana-ref}/kuery-query.html[Kibana Query Language] in your ES|QL queries: + +[source,esql] +---- +FROM logs* +| WHERE KQL("http.request.method:GET AND agent.type:filebeat") +---- + +The `kql` function is useful when transitioning queries from Kibana's Discover, Dashboard, or other interfaces that use KQL. This will allow you to gradually migrate queries to ES|QL without needing to rewrite them all at once. + +[[esql-for-search-semantic]] +==== Semantic search + +You can perform semantic searches over <> field types using the same match syntax as full-text search. + +This example uses the match operator `:`: + +[source,esql] +---- +FROM articles METADATA _score +| WHERE semantic_content:"What are the impacts of climate change on agriculture?" +| SORT _score DESC +---- + +This example uses the match function: + +[source,esql] +---- +FROM articles METADATA _score +| WHERE match(semantic_content, "What are the impacts of climate change on agriculture?") +| SORT _score DESC +---- + +[[esql-for-search-hybrid]] +==== Hybrid search + +Combine traditional and semantic search with custom weights: + +[source,esql] +---- +FROM books METADATA _score +| WHERE match(semantic_title, "fantasy adventure", { "boost": 0.75 }) + OR match(title, "fantasy adventure", { "boost": 0.25 }) +| SORT _score DESC +---- + +[[esql-for-search-limitations]] +==== Limitations + +Refer to <> for a list of known limitations. + +[[esql-for-search-next-steps]] +==== Next steps + +[[esql-for-search-tutorials]] +===== Tutorials and how-to guides + +* <>: Hands-on tutorial for getting started with search tools in {esql} +* <>: Learn how to use the `semantic_text` field type + +[[esql-for-search-reference]] +===== Technical reference + +* <>: Complete reference for all search functions +* <>: Current limitations for search in ES|QL + +[[esql-for-search-concepts]] +===== Background concepts + +* <>: Learn how text is processed for full-text search +* <>: Get an overview of semantic search in {es} +* <>: Understand the difference between query and filter contexts in {es} + +[[esql-for-search-blogs]] +===== Related blog posts + +// TODO* https://www.elastic.co/blog/esql-you-know-for-search-scoring-semantic-search[ES|QL, you know for Search]: Introducing scoring and semantic search +* https://www.elastic.co/search-labs/blog/filtering-in-esql-full-text-search-match-qstr[Introducing full text filtering in ES|QL]: Overview of text filtering capabilities diff --git a/docs/reference/esql/esql-get-started.asciidoc b/docs/reference/esql/esql-get-started.asciidoc index b7928898a3bbb..6451ed1909201 100644 --- a/docs/reference/esql/esql-get-started.asciidoc +++ b/docs/reference/esql/esql-get-started.asciidoc @@ -4,7 +4,7 @@ Getting started ++++ -This guide shows how you can use {esql} to query and aggregate your data. +This guide shows how you can use {esql} to query and aggregate your data. Refer to <> if you'd like to learn more about using {esql} for search use cases. [TIP] ==== diff --git a/docs/reference/esql/esql-kibana.asciidoc b/docs/reference/esql/esql-kibana.asciidoc index 87dd4d87fa8e3..5eb4f1d7a0446 100644 --- a/docs/reference/esql/esql-kibana.asciidoc +++ b/docs/reference/esql/esql-kibana.asciidoc @@ -37,7 +37,7 @@ To get started with {esql} in Discover, open the main menu and select [[esql-kibana-query-bar]] === The query bar -After switching to {esql} mode, the query bar shows a sample query. For example: +After switching to {esql} mode, the query bar shows your previous KQL or Lucene query converted into {esql}. If the query was empty, it shows a sample query. For example: [source,esql] ---- diff --git a/docs/reference/esql/esql-language.asciidoc b/docs/reference/esql/esql-language.asciidoc index 151ca803bf2eb..38960df404a37 100644 --- a/docs/reference/esql/esql-language.asciidoc +++ b/docs/reference/esql/esql-language.asciidoc @@ -11,6 +11,7 @@ Detailed reference documentation for the {esql} language: * <> * <> * <> +* <> * <> * <> * <> @@ -22,6 +23,7 @@ include::esql-functions-operators.asciidoc[] include::metadata-fields.asciidoc[] include::multivalued-fields.asciidoc[] include::esql-process-data-with-dissect-grok.asciidoc[] +include::esql-lookup-join.asciidoc[] include::esql-enrich-data.asciidoc[] include::implicit-casting.asciidoc[] include::time-spans.asciidoc[] diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 523330317b9f5..ddf0bf63a7dbd 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -9,7 +9,7 @@ [[esql-max-rows]] === Result set size limit -By default, an {esql} query returns up to 1000 rows. You can increase the number +By default, an {esql} query returns up to 1,000 rows. You can increase the number of rows up to 10,000 using the <> command. include::processing-commands/limit.asciidoc[tag=limitation] diff --git a/docs/reference/esql/esql-lookup-join.asciidoc b/docs/reference/esql/esql-lookup-join.asciidoc new file mode 100644 index 0000000000000..c78080258eb83 --- /dev/null +++ b/docs/reference/esql/esql-lookup-join.asciidoc @@ -0,0 +1,183 @@ +=== LOOKUP JOIN + +++++ +Correlate data with LOOKUP JOIN +++++ + +The {esql} <> +processing command combines data from your {esql} query results +table with matching records from a specified lookup index. It adds +fields from the lookup index as new columns to your results table based +on matching values in the join field. + +Teams often have data scattered across multiple indices – like logs, +IPs, user IDs, hosts, employees etc. Without a direct way to enrich or +correlate each event with reference data, root-cause analysis, security +checks, and operational insights become time-consuming. + +For example, you can use `LOOKUP JOIN` to: + +* Retrieve environment or ownership details for each host to correlate +your metrics data. +* Quickly see if any source IPs match known malicious addresses. +* Tag logs with the owning team or escalation info for faster triage and +incident response. + +<> is similar to <> +in the fact that they both help you join data together. You should use +`LOOKUP JOIN` when: + +* Your enrichment data changes frequently +* You want to avoid index-time processing +* You want SQL-like behavior, so that multiple matches result in multiple rows +* You need to match on any field in a lookup index +* You use document or field level security +* You want to restrict users to use only specific lookup indices +* You do not need to match using ranges or spatial relations + +[discrete] +[[esql-how-lookup-join-works]] +==== How the `LOOKUP JOIN` command works + +The `LOOKUP JOIN` command adds new columns to a table, with data from +{es} indices. + +image::images/esql/esql-lookup-join.png[align="center"] + +[[esql-lookup-join-lookup-index]] +lookup_index:: +The name of the lookup index. This must +be a specific index name - wildcards, aliases, and remote cluster +references are not supported. + +[[esql-lookup-join-field-name]] +field_name:: +The field to join on. This field must exist +in both your current query results and in the lookup index. If the field +contains multi-valued entries, those entries will not match anything +(the added fields will contain `null` for those rows). + +[discrete] +[[esql-lookup-join-example]] +==== Example + +`LOOKUP JOIN` has left-join behavior. If no rows match in the lookup index, `LOOKUP JOIN` retains the incoming row and adds nulls. If many rows in the lookup index match, `LOOKUP JOIN` adds one row per match. + +In this example, we have two sample tables: + +*employees* + +[cols=",,,,,",options="header",] +|=== +|birth++_++date |emp++_++no |first++_++name |gender |hire++_++date +|language +|1955-10-04T00:00:00Z |10091 |Amabile |M |1992-11-18T00:00:00Z |3 + +|1964-10-18T00:00:00Z |10092 |Valdiodio |F |1989-09-22T00:00:00Z |1 + +|1964-06-11T00:00:00Z |10093 |Sailaja |M |1996-11-05T00:00:00Z |3 + +|1957-05-25T00:00:00Z |10094 |Arumugam |F |1987-04-18T00:00:00Z |5 + +|1965-01-03T00:00:00Z |10095 |Hilari |M |1986-07-15T00:00:00Z |4 +|=== + +*languages++_++non++_++unique++_++key* + +[cols=",,",options="header",] +|=== +|language++_++code |language++_++name |country +|1 |English |Canada +|1 |English | +|1 | |United Kingdom +|1 |English |United States of America +|2 |German |++[++Germany{vbar}Austria++]++ +|2 |German |Switzerland +|2 |German | +|4 |Quenya | +|5 | |Atlantis +|++[++6{vbar}7++]++ |Mv-Lang |Mv-Land +|++[++7{vbar}8++]++ |Mv-Lang2 |Mv-Land2 +|Null-Lang |Null-Land | +|Null-Lang2 |Null-Land2 | +|=== + +Running the following query would provide the results shown below. + +[source,esql] +---- +FROM employees +| EVAL language_code = emp_no % 10 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| WHERE emp_no > 10090 AND emp_no < 10096 +| SORT emp_no, country +| KEEP emp_no, language_code, language_name, country; +---- + +[cols=",,,",options="header",] +|=== +|emp++_++no |language++_++code |language++_++name |country +|10091 |1 |English |Canada +|10091 |1 |null |United Kingdom +|10091 |1 |English |United States of America +|10091 |1 |English |null +|10092 |2 |German |++[++Germany, Austria++]++ +|10092 |2 |German |Switzerland +|10092 |2 |German |null +|10093 |3 |null |null +|10094 |4 |Spanish |null +|10095 |5 |null |France +|=== + +[IMPORTANT] +==== +`LOOKUP JOIN` does not guarantee the output to be in +any particular order. If a certain order is required, users should use a +<> somewhere after the `LOOKUP JOIN`. +==== + +[discrete] +[[esql-lookup-join-prereqs]] +==== Prerequisites + +To use `LOOKUP JOIN`, the following requirements must be met: + +* *Compatible data types*: The join key and join field in the lookup +index must have compatible data types. This means: +** The data types must either be identical or be internally represented +as the same type in {esql} +** Numeric types follow these compatibility rules: +*** `short` and `byte` are compatible with `integer` (all represented as +`int`) +*** `float`, `half_float`, and `scaled_float` are compatible +with `double` (all represented as `double`) +** For text fields: You can only use text fields as the join key on the +left-hand side of the join and only if they have a `.keyword` subfield + +To obtain a join key with a compatible type, use a +<> if needed. + +For a complete list of supported data types and their internal +representations, see the <>. + +[discrete] +[[esql-lookup-join-limitations]] +==== Limitations + +The following are the current limitations with `LOOKUP JOIN` + +* Indices in <> mode are always single-sharded. +* Cross cluster search is unsupported initially. Both source and lookup indices +must be local. +* Currently, only matching on equality is supported. +* `LOOKUP JOIN` can only use a single match field and a single index. +Wildcards, aliases, datemath, and datastreams are not supported. +* The name of the match field in +`LOOKUP JOIN lu++_++idx ON match++_++field` must match an existing field +in the query. This may require renames or evals to achieve. +* The query will circuit break if there are too many matching documents +in the lookup index, or if the documents are too large. More precisely, +`LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large +amount of heap space is needed if the matching documents from the lookup +index for a batch are multiple megabytes or larger. This is roughly the +same as for `ENRICH`. diff --git a/docs/reference/esql/esql-using.asciidoc b/docs/reference/esql/esql-using.asciidoc index d2e18bf1b91a3..a7b7ad1998ab9 100644 --- a/docs/reference/esql/esql-using.asciidoc +++ b/docs/reference/esql/esql-using.asciidoc @@ -1,27 +1,33 @@ [[esql-using]] == Using {esql} +This page is an overview of the various ways you can use {esql} across different Elastic interfaces and use cases. + <>:: -Information about using the <>. +Learn how to use the <>. + +<>:: +Learn how to use {esql} for search use cases. <>:: -Using {esql} in {kib} to query and aggregate your data, create visualizations, +Learn how to use {esql} in {kib} to query and aggregate your data, create visualizations, and set up alerts. <>:: -Using {esql} in {elastic-sec} to investigate events in Timeline, create +Learn how to use {esql} in {elastic-sec} to investigate events in Timeline, create detection rules, and build {esql} queries using Elastic AI Assistant. <>:: -Using {esql} to query multiple indexes and resolve field type mismatches. +Learn how to use {esql} to query multiple indexes and resolve field type mismatches. <>:: -Using {esql} to query across multiple clusters. +Learn how to use {esql} to query across multiple clusters. <>:: -Using the <> to list and cancel {esql} queries. +Learn how to use the <> to list and cancel {esql} queries. include::esql-rest.asciidoc[] +include::esql-for-search.asciidoc[] include::esql-kibana.asciidoc[] include::esql-security-solution.asciidoc[] include::esql-multi-index.asciidoc[] diff --git a/docs/reference/esql/functions/functionNamedParams/match.asciidoc b/docs/reference/esql/functions/functionNamedParams/match.asciidoc index 924a4be0efb0e..7b731730a21a3 100644 --- a/docs/reference/esql/functions/functionNamedParams/match.asciidoc +++ b/docs/reference/esql/functions/functionNamedParams/match.asciidoc @@ -6,15 +6,15 @@ |=== name | types | description fuzziness | [keyword] | Maximum edit distance allowed for matching. -auto_generate_synonyms_phrase_query | [boolean] | If true, match phrase queries are automatically created for multi-term synonyms. -analyzer | [keyword] | Analyzer used to convert the text in the query value into token. +auto_generate_synonyms_phrase_query | [boolean] | If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true. +analyzer | [keyword] | Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used. minimum_should_match | [integer] | Minimum number of clauses that must match for a document to be returned. -zero_terms_query | [keyword] | Number of beginning characters left unchanged for fuzzy matching. -boost | [float] | Floating point number used to decrease or increase the relevance scores of the query. -fuzzy_transpositions | [boolean] | If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). -fuzzy_rewrite | [keyword] | Method used to rewrite the query. See the rewrite parameter for valid values and more information. -prefix_length | [integer] | Number of beginning characters left unchanged for fuzzy matching. -lenient | [boolean] | If false, format-based errors, such as providing a text query value for a numeric field, are returned. -operator | [keyword] | Boolean logic used to interpret text in the query value. -max_expansions | [integer] | Maximum number of terms to which the query will expand. +zero_terms_query | [keyword] | Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none. +boost | [float] | Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0. +fuzzy_transpositions | [boolean] | If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true. +fuzzy_rewrite | [keyword] | Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default. +prefix_length | [integer] | Number of beginning characters left unchanged for fuzzy matching. Defaults to 0. +lenient | [boolean] | If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false. +operator | [keyword] | Boolean logic used to interpret text in the query value. Defaults to OR. +max_expansions | [integer] | Maximum number of terms to which the query will expand. Defaults to 50. |=== diff --git a/docs/reference/esql/functions/grouping-functions.asciidoc b/docs/reference/esql/functions/grouping-functions.asciidoc index 839320ce23392..3290e990bcaa9 100644 --- a/docs/reference/esql/functions/grouping-functions.asciidoc +++ b/docs/reference/esql/functions/grouping-functions.asciidoc @@ -9,8 +9,10 @@ The <> command supports these grouping functions: // tag::group_list[] * <> -* experimental:[] <> +* experimental:[] <> NOTE: Requires a https://www.elastic.co/subscriptions[platinum license]. // end::group_list[] include::layout/bucket.asciidoc[] + +NOTE: The `CATEGORIZE` function requires a https://www.elastic.co/subscriptions[platinum license]. include::layout/categorize.asciidoc[] diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json index 55cf2042db3ec..56d18ae24c2da 100644 --- a/docs/reference/esql/functions/kibana/definition/match.json +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -20,10 +20,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -45,10 +45,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -70,10 +70,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -95,10 +95,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -120,10 +120,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -145,10 +145,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -170,10 +170,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -195,10 +195,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -220,10 +220,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -245,10 +245,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -270,10 +270,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -295,10 +295,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -320,10 +320,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -345,10 +345,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -370,10 +370,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -395,10 +395,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -420,10 +420,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -445,10 +445,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -470,10 +470,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -495,10 +495,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -520,10 +520,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -545,10 +545,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -570,10 +570,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -595,10 +595,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -620,10 +620,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -645,10 +645,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -670,10 +670,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -695,10 +695,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, @@ -720,10 +720,10 @@ }, { "name" : "options", - "type" : "function named parameters", - "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba).'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand.'}", + "type" : "function_named_parameters", + "mapParams" : "{name='fuzziness', values=[AUTO, 1, 2], description='Maximum edit distance allowed for matching.'}, {name='auto_generate_synonyms_phrase_query', values=[true, false], description='If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true.'}, {name='analyzer', values=[standard], description='Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer mapped for the field. If no analyzer is mapped, the index’s default analyzer is used.'}, {name='minimum_should_match', values=[2], description='Minimum number of clauses that must match for a document to be returned.'}, {name='zero_terms_query', values=[none, all], description='Indicates whether all documents or none are returned if the analyzer removes all tokens, such as when using a stop filter. Defaults to none.'}, {name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0.'}, {name='fuzzy_transpositions', values=[true, false], description='If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to true.'}, {name='fuzzy_rewrite', values=[constant_score_blended, constant_score, constant_score_boolean, top_terms_blended_freqs_N, top_terms_boost_N, top_terms_N], description='Method used to rewrite the query. See the rewrite parameter for valid values and more information. If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of top_terms_blended_freqs_${max_expansions} by default.'}, {name='prefix_length', values=[1], description='Number of beginning characters left unchanged for fuzzy matching. Defaults to 0.'}, {name='lenient', values=[true, false], description='If false, format-based errors, such as providing a text query value for a numeric field, are returned. Defaults to false.'}, {name='operator', values=[AND, OR], description='Boolean logic used to interpret text in the query value. Defaults to OR.'}, {name='max_expansions', values=[50], description='Maximum number of terms to which the query will expand. Defaults to 50.'}", "optional" : true, - "description" : "Match additional options as <>. See <> for more information." + "description" : "(Optional) Match additional options as <>. See <> for more information." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/layout/categorize.asciidoc b/docs/reference/esql/functions/layout/categorize.asciidoc index 2ddc6bb31fae0..4075949ab4d12 100644 --- a/docs/reference/esql/functions/layout/categorize.asciidoc +++ b/docs/reference/esql/functions/layout/categorize.asciidoc @@ -4,7 +4,7 @@ [[esql-categorize]] === `CATEGORIZE` -preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] *Syntax* diff --git a/docs/reference/esql/functions/layout/kql.asciidoc b/docs/reference/esql/functions/layout/kql.asciidoc index ea9e4871f5e53..8cf2687b240c1 100644 --- a/docs/reference/esql/functions/layout/kql.asciidoc +++ b/docs/reference/esql/functions/layout/kql.asciidoc @@ -4,7 +4,7 @@ [[esql-kql]] === `KQL` -preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] *Syntax* diff --git a/docs/reference/esql/functions/layout/match.asciidoc b/docs/reference/esql/functions/layout/match.asciidoc index 6c67dfa96ef00..7765cc707e390 100644 --- a/docs/reference/esql/functions/layout/match.asciidoc +++ b/docs/reference/esql/functions/layout/match.asciidoc @@ -4,7 +4,7 @@ [[esql-match]] === `MATCH` -preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] *Syntax* diff --git a/docs/reference/esql/functions/layout/qstr.asciidoc b/docs/reference/esql/functions/layout/qstr.asciidoc index b46c6498673b5..715a11089f0d4 100644 --- a/docs/reference/esql/functions/layout/qstr.asciidoc +++ b/docs/reference/esql/functions/layout/qstr.asciidoc @@ -4,7 +4,7 @@ [[esql-qstr]] === `QSTR` -preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] *Syntax* diff --git a/docs/reference/esql/functions/layout/term.asciidoc b/docs/reference/esql/functions/layout/term.asciidoc index 2007535dc2c74..1fe94491bed04 100644 --- a/docs/reference/esql/functions/layout/term.asciidoc +++ b/docs/reference/esql/functions/layout/term.asciidoc @@ -4,7 +4,7 @@ [[esql-term]] === `TERM` -preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] *Syntax* diff --git a/docs/reference/esql/functions/layout/values.asciidoc b/docs/reference/esql/functions/layout/values.asciidoc index 7d90d4314699a..0e80d2b89d3d9 100644 --- a/docs/reference/esql/functions/layout/values.asciidoc +++ b/docs/reference/esql/functions/layout/values.asciidoc @@ -4,7 +4,7 @@ [[esql-values]] === `VALUES` -preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] *Syntax* diff --git a/docs/reference/esql/functions/parameters/match.asciidoc b/docs/reference/esql/functions/parameters/match.asciidoc index bced59947622e..5ded9745025ab 100644 --- a/docs/reference/esql/functions/parameters/match.asciidoc +++ b/docs/reference/esql/functions/parameters/match.asciidoc @@ -9,4 +9,4 @@ Field that the query will target. Value to find in the provided field. `options`:: -Match additional options as <>. See <> for more information. +(Optional) Match additional options as <>. See <> for more information. diff --git a/docs/reference/esql/index.asciidoc b/docs/reference/esql/index.asciidoc index 54627a6de3c62..7c8c793d483bb 100644 --- a/docs/reference/esql/index.asciidoc +++ b/docs/reference/esql/index.asciidoc @@ -22,21 +22,7 @@ a series of operations, where the output of one operation becomes the input for the next, enabling complex data transformations and analysis. [discrete] -=== The {esql} Compute Engine - -{esql} is more than a language: it represents a significant investment in new -compute capabilities within {es}. To achieve both the functional and performance -requirements for {esql}, it was necessary to build an entirely new compute -architecture. {esql} search, aggregation, and transformation functions are -directly executed within Elasticsearch itself. Query expressions are not -transpiled to Query DSL for execution. This approach allows {esql} to be -extremely performant and versatile. - -The new {esql} execution engine was designed with performance in mind — it -operates on blocks at a time instead of per row, targets vectorization and cache -locality, and embraces specialization and multi-threading. It is a separate -component from the existing Elasticsearch aggregation framework with different -performance characteristics. +=== Documentation organization The {esql} documentation is organized in these sections: @@ -45,16 +31,24 @@ A tutorial to help you get started with {esql}. <>:: -Reference documentation for the <>, -<>, and <>. Information about working with <> and <>. And guidance for -<> and <>. +Reference documentation for the <>: + +* Reference for <>, and <> +* How to work with <> and <> +* How to work with +<>, <>, and <> <>:: -An overview of using the <>, <>, -<>, <>, and <>. +An overview of: +* <> +* <> +* <> +* <> +* <> +* <> <>:: The current limitations of {esql}. @@ -62,6 +56,8 @@ The current limitations of {esql}. <>:: A few examples of what you can do with {esql}. + + include::esql-get-started.asciidoc[] include::esql-language.asciidoc[] @@ -74,3 +70,20 @@ include::esql-examples.asciidoc[] :esql-tests!: :esql-specs!: + +[discrete] +=== The {esql} Compute Engine + +{esql} is more than a language: it represents a significant investment in new +compute capabilities within {es}. To achieve both the functional and performance +requirements for {esql}, it was necessary to build an entirely new compute +architecture. {esql} search, aggregation, and transformation functions are +directly executed within Elasticsearch itself. Query expressions are not +transpiled to Query DSL for execution. This approach allows {esql} to be +extremely performant and versatile. + +The new {esql} execution engine was designed with performance in mind — it +operates on blocks at a time instead of per row, targets vectorization and cache +locality, and embraces specialization and multi-threading. It is a separate +component from the existing Elasticsearch aggregation framework with different +performance characteristics. \ No newline at end of file diff --git a/docs/reference/esql/processing-commands/limit.asciidoc b/docs/reference/esql/processing-commands/limit.asciidoc index 78d05672ea095..2ef31a59f44b7 100644 --- a/docs/reference/esql/processing-commands/limit.asciidoc +++ b/docs/reference/esql/processing-commands/limit.asciidoc @@ -22,24 +22,54 @@ The maximum number of rows to return. The `LIMIT` processing command enables you to limit the number of rows that are returned. // tag::limitation[] -Queries do not return more than 10,000 rows, regardless of the `LIMIT` command's -value. +For instance, +``` +FROM index | WHERE field = "value" +``` +is equivalent to: +``` +FROM index | WHERE field = "value" | LIMIT 1000 +``` -This limit only applies to the number of rows that are retrieved by the query. -Queries and aggregations run on the full data set. +Queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. This is a configurable upper limit. To overcome this limitation: -* Reduce the result set size by modifying the query to only return relevant -data. Use <> to select a smaller subset of the data. -* Shift any post-query processing to the query itself. You can use the {esql} -<> command to aggregate data in the query. +* Reduce the result set size by modifying the query to only return relevant data. Use <> to select a smaller subset of the data. +* Shift any post-query processing to the query itself. You can use the {esql} <> command to aggregate data in the query. -The default and maximum limits can be changed using these dynamic cluster -settings: +The upper limit only applies to the number of rows that are output by the query, not to the number of documents it processes: the query runs on the full data set. + +Consider the following two queries: +``` +FROM index | WHERE field0 == "value" | LIMIT 20000 +``` +and +``` +FROM index | STATS AVG(field1) BY field2 | LIMIT 20000 +``` + +In both cases, the filtering by `field0` in the first query or the grouping by `field2` in the second is applied over all the documents present in the `index`, irrespective of their number or indexes size. However, both queries will return at most 10,000 rows, even if there were more rows available to return. + +The default and maximum limits can be changed using these dynamic cluster settings: * `esql.query.result_truncation_default_size` * `esql.query.result_truncation_max_size` + +However, doing so involves trade-offs. A larger result-set involves a higher memory pressure and increased processing times; the internode traffic within and across clusters can also increase. + +These limitations are similar to those enforced by the <>. + +[%header.monospaced.styled,format=dsv,separator=|] +|=== + +Functionality | Search | {esql} +Results returned by default | 10 | 1.000 +Default upper limit | 10,000 | 10,000 +Specify number of results | `size` | `LIMIT` +Change default number of results | n/a | esql.query.result_truncation_default_size +Change default upper limit | index-max-result-window | esql.query.result_truncation_max_size +|=== // end::limitation[] *Example* diff --git a/docs/reference/esql/processing-commands/lookup.asciidoc b/docs/reference/esql/processing-commands/lookup.asciidoc new file mode 100644 index 0000000000000..cde5130a68815 --- /dev/null +++ b/docs/reference/esql/processing-commands/lookup.asciidoc @@ -0,0 +1,114 @@ +[discrete] +[[esql-lookup-join]] +=== `LOOKUP JOIN` + +[WARNING] +==== +This functionality is in technical preview and may be +changed or removed in a future release. Elastic will work to fix any +issues, but features in technical preview are not subject to the support +SLA of official GA features. +==== +`LOOKUP JOIN` enables you to add data from another index, AKA a 'lookup' +index, to your {esql} query results, simplifying data enrichment +and analysis workflows. + +*Syntax* + +[source,esql] +---- +FROM +| LOOKUP JOIN ON +---- + +*Parameters* + +`lookup_index`:: +The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster +references are not supported. + +`field_name`:: +The field to join on. This field must exist +in both your current query results and in the lookup index. If the field +contains multi-valued entries, those entries will not match anything +(the added fields will contain `null` for those rows). + +*Description* + +The `LOOKUP JOIN` command adds new columns to your {esql} query +results table by finding documents in a lookup index that share the same +join field value as your result rows. + +For each row in your results table that matches a document in the lookup +index based on the join field, all fields from the matching document are +added as new columns to that row. + +If multiple documents in the lookup index match a single row in your +results, the output will contain one row for each matching combination. + +*Examples* + +[TIP] +==== +In case of name collisions, the newly created columns will override existing columns. +==== + +*IP Threat correlation*: This query would allow you to see if any source +IPs match known malicious addresses. + +[source,esql] +---- +FROM firewall_logs +| LOOKUP JOIN threat_list ON source.IP +---- + +To filter only for those rows that have a matching `threat_list` entry, use `WHERE ... IS NOT NULL` with a field from the lookup index: + +[source,esql] +---- +FROM firewall_logs +| LOOKUP JOIN threat_list ON source.IP +| WHERE threat_level IS NOT NULL +---- + +*Host metadata correlation*: This query pulls in environment or +ownership details for each host to correlate with your metrics data. + +[source,esql] +---- +FROM system_metrics +| LOOKUP JOIN host_inventory ON host.name +| LOOKUP JOIN employees ON host.name +---- + +*Service ownership mapping*: This query would show logs with the owning +team or escalation information for faster triage and incident response. + +[source,esql] +---- +FROM app_logs +| LOOKUP JOIN service_owners ON service_id +---- + +`LOOKUP JOIN` is generally faster when there are fewer rows to join +with. {esql} will try and perform any `WHERE` clause before the +`LOOKUP JOIN` where possible. + +The two following examples will have the same results. The two examples +have the `WHERE` clause before and after the `LOOKUP JOIN`. It does not +matter how you write your query, our optimizer will move the filter +before the lookup when ran. + +[source,esql] +---- +FROM Left +| WHERE Language IS NOT NULL +| LOOKUP JOIN Right ON Key +---- + +[source,esql] +---- +FROM Left +| LOOKUP JOIN Right ON Key +| WHERE Language IS NOT NULL +---- diff --git a/docs/reference/esql/processing-commands/lookup.disabled b/docs/reference/esql/processing-commands/lookup.disabled deleted file mode 100644 index ca456d8e70eed..0000000000000 --- a/docs/reference/esql/processing-commands/lookup.disabled +++ /dev/null @@ -1,64 +0,0 @@ -[discrete] -[[esql-lookup]] -=== `LOOKUP` - -experimental::["LOOKUP is highly experimental and only available in SNAPSHOT versions."] - -`LOOKUP` matches values from the input against a `table` provided in the request, -adding the other fields from the `table` to the output. - -**Syntax** - -[source,esql] ----- -LOOKUP table ON match_field1[, match_field2, ...] ----- - -*Parameters* - -`table`:: -The name of the `table` provided in the request to match. -If the table's column names conflict with existing columns, the existing columns will be dropped. - -`match_field`:: -The fields in the input to match against the table. - -*Examples* - -// tag::examples[] -[source,console,id=esql-lookup-example] ----- -POST /_query?format=txt -{ - "query": """ - FROM library - | SORT page_count DESC - | KEEP name, author - | LOOKUP era ON author - | LIMIT 5 - """, - "tables": { - "era": { - "author": {"keyword": ["Frank Herbert", "Peter F. Hamilton", "Vernor Vinge", "Alastair Reynolds", "James S.A. Corey"]}, - "era": {"keyword": [ "The New Wave", "Diamond", "Diamond", "Diamond", "Hadron"]} - } - } -} ----- -// TEST[setup:library] - -Which returns: - -[source,text] ----- - name | author | era ---------------------+-----------------+--------------- -Pandora's Star |Peter F. Hamilton|Diamond -A Fire Upon the Deep|Vernor Vinge |Diamond -Dune |Frank Herbert |The New Wave -Revelation Space |Alastair Reynolds|Diamond -Leviathan Wakes |James S.A. Corey |Hadron ----- -// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/] -// TESTRESPONSE[non_json] -// end::examples[] diff --git a/docs/reference/features/apis/features-apis.asciidoc b/docs/reference/features/apis/features-apis.asciidoc index 2582446340f15..0e327f974bc9a 100644 --- a/docs/reference/features/apis/features-apis.asciidoc +++ b/docs/reference/features/apis/features-apis.asciidoc @@ -13,7 +13,7 @@ by Elasticsearch and Elasticsearch plugins. [discrete] === Features APIs * <> -* <> +* <> include::get-features-api.asciidoc[] include::reset-features-api.asciidoc[] diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc index 60c32cabdb5c1..a65167b8d36c2 100644 --- a/docs/reference/how-to/knn-search.asciidoc +++ b/docs/reference/how-to/knn-search.asciidoc @@ -76,7 +76,7 @@ Here are estimates for different element types and quantization levels: * `element_type: float`: `num_vectors * num_dimensions * 4` * `element_type: float` with `quantization: int8`: `num_vectors * (num_dimensions + 4)` * `element_type: float` with `quantization: int4`: `num_vectors * (num_dimensions/2 + 4)` -* `element_type: float` with `quantization: bbq`: `num_vectors * (num_dimensions/8 + 12)` +* `element_type: float` with `quantization: bbq`: `num_vectors * (num_dimensions/8 + 14)` * `element_type: byte`: `num_vectors * num_dimensions` * `element_type: bit`: `num_vectors * (num_dimensions/8)` diff --git a/docs/reference/images/analysis/token-graph-dns-synonym-ex2.svg b/docs/reference/images/analysis/token-graph-dns-synonym-ex2.svg new file mode 100644 index 0000000000000..4fe42552a3eb2 --- /dev/null +++ b/docs/reference/images/analysis/token-graph-dns-synonym-ex2.svg @@ -0,0 +1,2 @@ +domainnamesystemisfragileinternetphonebook02345⊙1 diff --git a/docs/reference/images/analysis/token-graph-dns-synonym-flattened-ex2.svg b/docs/reference/images/analysis/token-graph-dns-synonym-flattened-ex2.svg new file mode 100644 index 0000000000000..b7faa6570ad83 --- /dev/null +++ b/docs/reference/images/analysis/token-graph-dns-synonym-flattened-ex2.svg @@ -0,0 +1,2 @@ +domainnamesystemisfragileinternetphonebook01234⊙ diff --git a/docs/reference/images/esql/esql-lookup-join.png b/docs/reference/images/esql/esql-lookup-join.png new file mode 100644 index 0000000000000..de220b0638a06 Binary files /dev/null and b/docs/reference/images/esql/esql-lookup-join.png differ diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index 73e2db6e45e34..e431a1bb6e1aa 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -113,6 +113,8 @@ Index mode supports the following values: `standard`::: Standard indexing with default settings. +`lookup`::: Index that can be used for lookup joins in ES|QL. Limited to 1 shard. + `time_series`::: _(data streams only)_ Index mode optimized for storage of metrics. For more information, see <>. `logsdb`::: _(data streams only)_ Index mode optimized for <>. diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc index e848668c1a66d..a0c0a5e965818 100644 --- a/docs/reference/index-modules/slowlog.asciidoc +++ b/docs/reference/index-modules/slowlog.asciidoc @@ -107,6 +107,7 @@ GET _all/_settings?expand_wildcards=all&filter_path=*.settings.index.*.slowlog [[search-slow-log]] ==== Enable slow logging for search events +[[_identifying_search_slow_log_origin]] Search slow logs emit per shard. They must be enabled separately for the shard's link:https://www.elastic.co/blog/understanding-query-then-fetch-vs-dfs-query-then-fetch[query and fetch search phases]. You can use the `index.search.slowlog.include.user` setting to append `user.*` and `auth.type` fields to slow log entries. These fields contain information about the user who triggered the request. diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc index 479bdff22a80c..a377f92d6cfd3 100644 --- a/docs/reference/indices/put-mapping.asciidoc +++ b/docs/reference/indices/put-mapping.asciidoc @@ -10,8 +10,14 @@ For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. -- -Adds new fields to an existing data stream or index. You can also use this -API to change the search settings of existing fields. +You can use this API to: + +* Add new fields to an existing index +* Change the search settings of existing fields + +NOTE: Elasticsearch does not allow in-place field type changes. If you need to change a field’s type, you must create a new index with the updated mapping and <> the data. + + For data streams, these changes are applied to all backing indices by default. diff --git a/docs/reference/indices/resolve-cluster.asciidoc b/docs/reference/indices/resolve-cluster.asciidoc index 195cbb997adb1..19f119dff1624 100644 --- a/docs/reference/indices/resolve-cluster.asciidoc +++ b/docs/reference/indices/resolve-cluster.asciidoc @@ -101,6 +101,10 @@ about whether it has any indices, aliases or data streams that match `my-index-* [[resolve-cluster-api-request]] ==== {api-request-title} +`GET /_resolve/cluster` + +or + `GET /_resolve/cluster/` [[resolve-cluster-api-prereqs]] @@ -116,7 +120,7 @@ privilege>> for the target data stream, index, or alias. ``:: + -- -(Required, string) Comma-separated name(s) or index pattern(s) of the +(Optional, string) Comma-separated name(s) or index pattern(s) of the indices, aliases, and data streams to resolve, using <>. Resources on <> can be specified using the `:` syntax. diff --git a/docs/reference/inference/chat-completion-inference.asciidoc b/docs/reference/inference/chat-completion-inference.asciidoc index 1d7d05b0f7d82..88699cca67af4 100644 --- a/docs/reference/inference/chat-completion-inference.asciidoc +++ b/docs/reference/inference/chat-completion-inference.asciidoc @@ -13,9 +13,9 @@ However, if you do not plan to use the {infer} APIs to use these models or if yo [[chat-completion-inference-api-request]] ==== {api-request-title} -`POST /_inference//_unified` +`POST /_inference//_stream` -`POST /_inference/chat_completion//_unified` +`POST /_inference/chat_completion//_stream` [discrete] @@ -37,7 +37,7 @@ It only works with the `chat_completion` task type for `openai` and `elastic` {i [NOTE] ==== -* The `chat_completion` task type is only available within the _unified API and only supports streaming. +* The `chat_completion` task type is only available within the _stream API and only supports streaming. * The Chat completion {infer} API and the Stream {infer} API differ in their response structure and capabilities. The Chat completion {infer} API provides more comprehensive customization options through more fields and function calling support. If you use the `openai` service or the `elastic` service, use the Chat completion {infer} API. diff --git a/docs/reference/inference/elastic-infer-service.asciidoc b/docs/reference/inference/elastic-infer-service.asciidoc deleted file mode 100644 index 24ae7e20deec6..0000000000000 --- a/docs/reference/inference/elastic-infer-service.asciidoc +++ /dev/null @@ -1,124 +0,0 @@ -[[infer-service-elastic]] -=== Elastic {infer-cap} Service (EIS) - -.New API reference -[sidebar] --- -For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. --- - -Creates an {infer} endpoint to perform an {infer} task with the `elastic` service. - - -[discrete] -[[infer-service-elastic-api-request]] -==== {api-request-title} - - -`PUT /_inference//` - -[discrete] -[[infer-service-elastic-api-path-params]] -==== {api-path-parms-title} - - -``:: -(Required, string) -include::inference-shared.asciidoc[tag=inference-id] - -``:: -(Required, string) -include::inference-shared.asciidoc[tag=task-type] -+ --- -Available task types: - -* `chat_completion`, -* `sparse_embedding`. --- - -[NOTE] -==== -The `chat_completion` task type only supports streaming and only through the `_unified` API. - -include::inference-shared.asciidoc[tag=chat-completion-docs] -==== - -[discrete] -[[infer-service-elastic-api-request-body]] -==== {api-request-body-title} - - -`max_chunk_size`::: -(Optional, integer) -include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] - -`overlap`::: -(Optional, integer) -include::inference-shared.asciidoc[tag=chunking-settings-overlap] - -`sentence_overlap`::: -(Optional, integer) -include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] - -`strategy`::: -(Optional, string) -include::inference-shared.asciidoc[tag=chunking-settings-strategy] - -`service`:: -(Required, string) -The type of service supported for the specified task type. In this case, -`elastic`. - -`service_settings`:: -(Required, object) -include::inference-shared.asciidoc[tag=service-settings] - -`model_id`::: -(Required, string) -The name of the model to use for the {infer} task. - -`rate_limit`::: -(Optional, object) -By default, the `elastic` service sets the number of requests allowed per minute to `1000` in case of `sparse_embedding` and `240` in case of `chat_completion`. -This helps to minimize the number of rate limit errors returned. -To modify this, set the `requests_per_minute` setting of this object in your service settings: -+ --- -include::inference-shared.asciidoc[tag=request-per-minute-example] --- - - -[discrete] -[[inference-example-elastic]] -==== Elastic {infer-cap} Service example - - -The following example shows how to create an {infer} endpoint called `elser-model-eis` to perform a `text_embedding` task type. - -[source,console] ------------------------------------------------------------- -PUT _inference/sparse_embedding/elser-model-eis -{ - "service": "elastic", - "service_settings": { - "model_name": "elser" - } -} - ------------------------------------------------------------- -// TEST[skip:TBD] - -The following example shows how to create an {infer} endpoint called `chat-completion-endpoint` to perform a `chat_completion` task type. - -[source,console] ------------------------------------------------------------- -PUT /_inference/chat_completion/chat-completion-endpoint -{ - "service": "elastic", - "service_settings": { - "model_id": "model-1" - } -} ------------------------------------------------------------- -// TEST[skip:TBD] \ No newline at end of file diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index aa1d54de60391..9c5218324f229 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -58,8 +58,10 @@ For more information about adaptive allocations and resources, refer to the {ml- Your {es} deployment contains preconfigured {infer} endpoints which makes them easier to use when defining `semantic_text` fields or using {infer} processors. The following list contains the default {infer} endpoints listed by `inference_id`: -* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts) -* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts) +* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language tex). +The `model_id` is `.elser_model_2_linux-x86_64`. +* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts). +The `model_id` is `.e5_model_2_linux-x86_64`. Use the `inference_id` of the endpoint in a <> field definition or when creating an <>. The API call will automatically download and deploy the model which might take a couple of minutes. @@ -136,7 +138,6 @@ include::chat-completion-inference.asciidoc[] include::put-inference.asciidoc[] include::stream-inference.asciidoc[] include::update-inference.asciidoc[] -include::elastic-infer-service.asciidoc[] include::service-alibabacloud-ai-search.asciidoc[] include::service-amazon-bedrock.asciidoc[] include::service-anthropic.asciidoc[] @@ -151,4 +152,5 @@ include::service-hugging-face.asciidoc[] include::service-jinaai.asciidoc[] include::service-mistral.asciidoc[] include::service-openai.asciidoc[] +include::service-voyageai.asciidoc[] include::service-watsonx-ai.asciidoc[] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 6e33619c11e59..73b036ef6880b 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -59,8 +59,6 @@ The create {infer} API enables you to create an {infer} endpoint and configure a * Avoid creating multiple endpoints for the same model unless required, as each endpoint consumes significant resources. ==== -You can create an {infer} endpoint that uses the <> to perform {infer} tasks as a service without the need of deploying a model in your environment. - The following integrations are available through the {infer} API. You can find the available task types next to the integration name. Click the links to review the configuration details of the integrations: @@ -78,6 +76,7 @@ Click the links to review the configuration details of the integrations: * <> (`text_embedding`) * <> (`text_embedding`) * <> (`chat_completion`, `completion`, `text_embedding`) +* <> (`text_embedding`, `rerank`) * <> (`text_embedding`) * <> (`text_embedding`, `rerank`) diff --git a/docs/reference/inference/service-openai.asciidoc b/docs/reference/inference/service-openai.asciidoc index 511632736a35b..9bc673f941b33 100644 --- a/docs/reference/inference/service-openai.asciidoc +++ b/docs/reference/inference/service-openai.asciidoc @@ -7,7 +7,7 @@ For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. -- -Creates an {infer} endpoint to perform an {infer} task with the `openai` service. +Creates an {infer} endpoint to perform an {infer} task with the `openai` service or `openai` compatible APIs. [discrete] @@ -38,7 +38,7 @@ Available task types: [NOTE] ==== -The `chat_completion` task type only supports streaming and only through the `_unified` API. +The `chat_completion` task type only supports streaming and only through the `_stream` API. include::inference-shared.asciidoc[tag=chat-completion-docs] ==== diff --git a/docs/reference/inference/service-voyageai.asciidoc b/docs/reference/inference/service-voyageai.asciidoc new file mode 100644 index 0000000000000..549f18dd5a011 --- /dev/null +++ b/docs/reference/inference/service-voyageai.asciidoc @@ -0,0 +1,178 @@ +[[infer-service-voyageai]] +=== VoyageAI {infer} integration + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + +Creates an {infer} endpoint to perform an {infer} task with the `voyageai` service. + + +[discrete] +[[infer-service-voyageai-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-voyageai-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `text_embedding`, +* `rerank`. +-- + +[discrete] +[[infer-service-voyageai-api-request-body]] +==== {api-request-body-title} + +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunk_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + +`service`:: +(Required, string) +The type of service supported for the specified task type. In this case, +`voyageai`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `voyageai` service. +-- + +`dimensions`::: +(Optional, integer) +The number of dimensions the resulting output embeddings should have. +This setting maps to `output_dimension` in the https://docs.voyageai.com/docs/embeddings[VoyageAI documentation]. +Only for the `text_embedding` task type. + +`embedding_type`::: +(Optional, string) +The data type for the embeddings to be returned. +This setting maps to `output_dtype` in the https://docs.voyageai.com/docs/embeddings[VoyageAI documentation]. +Permitted values: `float`, `int8`, `bit`. +`int8` is a synonym of `byte` in the VoyageAI documentation. +`bit` is a synonym of `binary` in the VoyageAI documentation. +Only for the `text_embedding` task type. + +`model_id`::: +(Required, string) +The name of the model to use for the {infer} task. +Refer to the VoyageAI documentation for the list of available https://docs.voyageai.com/docs/embeddings[text embedding] and https://docs.voyageai.com/docs/reranker[rerank] models. + +`rate_limit`::: +(Optional, object) +This setting helps to minimize the number of rate limit errors returned from VoyageAI. +The `voyageai` service sets a default number of requests allowed per minute depending on the task type. +For both `text_embedding` and `rerank`, it is set to `2000`. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] + +More information about the rate limits for OpenAI can be found in your https://platform.openai.com/account/limits[Account limits]. +-- + +`task_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `text_embedding` task type +[%collapsible%closed] +===== +`input_type`::: +(Optional, string) +Type of the input text. +Permitted values: `ingest` (maps to `document` in the VoyageAI documentation), `search` (maps to `query` in the VoyageAI documentation). + +`truncation`::: +(Optional, boolean) +Whether to truncate the input texts to fit within the context length. +Defaults to `false`. +===== ++ +.`task_settings` for the `rerank` task type +[%collapsible%closed] +===== +`return_documents`::: +(Optional, boolean) +Whether to return the source documents in the response. +Defaults to `false`. + +`top_k`::: +(Optional, integer) +The number of most relevant documents to return. +If not specified, the reranking results of all documents will be returned. + +`truncation`::: +(Optional, boolean) +Whether to truncate the input texts to fit within the context length. +Defaults to `false`. +===== + + +[discrete] +[[inference-example-voyageai]] +==== VoyageAI service example + +The following example shows how to create an {infer} endpoint called `voyageai-embeddings` to perform a `text_embedding` task type. +The embeddings created by requests to this endpoint will have 512 dimensions. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/voyageai-embeddings +{ + "service": "voyageai", + "service_settings": { + "model_id": "voyage-3-large", + "dimensions": 512 + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The next example shows how to create an {infer} endpoint called `voyageai-rerank` to perform a `rerank` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/voyageai-rerank +{ + "service": "voyageai", + "service_settings": { + "model_id": "rerank-2" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/inference/service-watsonx-ai.asciidoc b/docs/reference/inference/service-watsonx-ai.asciidoc index 31d246a36d350..5a74265ad6306 100644 --- a/docs/reference/inference/service-watsonx-ai.asciidoc +++ b/docs/reference/inference/service-watsonx-ai.asciidoc @@ -34,7 +34,8 @@ include::inference-shared.asciidoc[tag=task-type] -- Available task types: -* `text_embedding`. +* `text_embedding`, +* `rerank`. -- [discrete] @@ -91,6 +92,26 @@ To modify this, set the `requests_per_minute` setting of this object in your ser include::inference-shared.asciidoc[tag=request-per-minute-example] -- +`task_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `rerank` task type +[%collapsible%closed] +===== +`truncate_input_tokens`::: +(Optional, integer) +Specifies the maximum number of tokens per input document before truncation. + +`return_documents`::: +(Optional, boolean) +Specify whether to return doc text within the results. + +`top_n`::: +(Optional, integer) +The number of most relevant documents to return. Defaults to the number of input documents. +===== + [discrete] [[inference-example-watsonx-ai]] @@ -118,4 +139,35 @@ PUT _inference/text_embedding/watsonx-embeddings You can find on the https://cloud.ibm.com/iam/apikeys[API keys page of your account]. <2> The {infer} endpoint URL you created on Watsonx. <3> The ID of your IBM Cloud project. -<4> A valid API version parameter. You can find the active version data parameters https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[here]. \ No newline at end of file +<4> A valid API version parameter. You can find the active version data parameters https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[here]. + +The following example shows how to create an {infer} endpoint called `watsonx-rerank` to perform a `rerank` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/watsonx-rerank +{ + "service": "watsonxai", + "service_settings": { + "api_key": "", <1> + "url": "", <2> + "model_id": "cross-encoder/ms-marco-minilm-l-12-v2", + "project_id": "", <3> + "api_version": "2024-05-02" <4> + }, + "task_settings": { + "truncate_input_tokens": 50, <5> + "return_documents": true, <6> + "top_n": 3 <7> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> A valid Watsonx API key. +You can find on the https://cloud.ibm.com/iam/apikeys[API keys page of your account]. +<2> The {infer} endpoint URL you created on Watsonx. +<3> The ID of your IBM Cloud project. +<4> A valid API version parameter. You can find the active version data parameters https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[here]. +<5> The maximum number of tokens per document before truncation. +<6> Whether to return the document text in the results. +<7> The number of top relevant documents to return. \ No newline at end of file diff --git a/docs/reference/ingest/processors.asciidoc b/docs/reference/ingest/processors.asciidoc index f4fcc0fc84d0d..55de1a8bee1a7 100644 --- a/docs/reference/ingest/processors.asciidoc +++ b/docs/reference/ingest/processors.asciidoc @@ -89,7 +89,7 @@ Calculates the network direction given a source IP address, destination IP addre Extracts the registered domain (also known as the effective top-level domain or eTLD), sub-domain, and top-level domain from a fully qualified domain name (FQDN). <>:: -Sets user-related details (such as `username`, `roles`, `email`, `full_name`,`metadata`, `api_key`, `realm` and `authentication_type`) from the current authenticated user to the current document by pre-processing the ingest. +Sets user-related details (such as `username`, `roles`, `email`, `full_name`,`metadata`, `api_key`, `realm` and `authentication_type`) from the current authenticated user. <>:: Parses a Uniform Resource Identifier (URI) string and extracts its components as an object. @@ -144,7 +144,7 @@ Extracts a single line of CSV data from a text field. <>:: Extracts and converts date fields. -<> processor:: +<>:: Expands a field with dots into an object field. <>:: diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 78ebe3f5b5ee3..8246d881229e6 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -5,7 +5,8 @@ ++++ The `geoip` processor adds information about the geographical location of an -IPv4 or IPv6 address. +IPv4 or IPv6 address. It is compatible with the GeoIP2 and GeoLite2 IP geolocation +databases available from https://www.maxmind.com/[MaxMind]. [[geoip-automatic-updates]] By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2 diff --git a/docs/reference/ingest/processors/ip-location.asciidoc b/docs/reference/ingest/processors/ip-location.asciidoc index e2ca9dbbe2eb3..e4e5f4a95db7b 100644 --- a/docs/reference/ingest/processors/ip-location.asciidoc +++ b/docs/reference/ingest/processors/ip-location.asciidoc @@ -5,7 +5,9 @@ ++++ The `ip_location` processor adds information about the geographical location of an -IPv4 or IPv6 address. +IPv4 or IPv6 address. It is compatible with the GeoIP2 and GeoLite2 IP geolocation +databases available from https://www.maxmind.com/[MaxMind], as well as many of the +IP geolocation databases available from https://ipinfo.io[IPinfo]. [[ip-location-automatic-updates]] By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2 @@ -24,8 +26,8 @@ stats API>>. If your cluster can't connect to the Elastic GeoIP endpoint or you want to manage your own updates, see <>. -If you would like to have {es} download database files directly from Maxmind using your own provided -license key, see <>. +If you would like to have {es} download database files directly from Maxmind or IPinfo using your own +license key or access token, see <>. If {es} can't connect to the endpoint for 30 days all updated databases will become invalid. {es} will stop enriching documents with ip geolocation data and will add `tags: ["_ip_location_expired_database"]` @@ -77,7 +79,20 @@ depend on what has been found and which properties were configured in `propertie `location`, `accuracy_radius`, `country_confidence`, `city_confidence`, `postal_confidence`, `asn`, `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`, `domain`, `isp`, `isp_organization_name`, `mobile_country_code`, `mobile_network_code`, `user_type`, and -`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties`. +`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties` +* If the IPinfo ASN database is used, then the following fields may be added under the `target_field`: `ip`, +`asn`, `organization_name`, `network`, `domain`, `country_iso_code`, and `type`. The fields actually added depend on what +has been found and which properties were configured in `properties`. Note also that `country_iso_code` and `type` fields +are only present in the 'Standard ASN' database, not the free ASN database. +* If the IPinfo Country database is used, then the following fields may be added under the `target_field`: `ip`, +`country_iso_code`, `country_name`, `continent_code`, and `continent_name`. The fields actually added depend on what +has been found and which properties were configured in `properties`. +* If the IPinfo IP to Location database is used, then the following fields may be added under the `target_field`: `ip`, +`country_iso_code`, `region_name`, `city_name`, `timezone`, `postal_code`, and `location`. The fields actually added depend on what +has been found and which properties were configured in `properties`. +* If the IPinfo Privacy Detection database is used, then the following fields may be added under the `target_field`: `ip`, +`hosting`, `proxy`, `relay`, `tor`, `vpn`, and `service`. The fields actually added depend on what +has been found and which properties were configured in `properties`. Here is an example that uses the default city database and adds the geographical information to the `ip_location` field based on the `ip` field: diff --git a/docs/reference/ingest/processors/set-security-user.asciidoc b/docs/reference/ingest/processors/set-security-user.asciidoc index 3213157827627..960525033bbae 100644 --- a/docs/reference/ingest/processors/set-security-user.asciidoc +++ b/docs/reference/ingest/processors/set-security-user.asciidoc @@ -4,9 +4,9 @@ Set security user ++++ -Sets user-related details (such as `username`, `roles`, `email`, `full_name`, +Sets user-related details (such as `username`, `roles`, `email`, `full_name`, `metadata`, `api_key`, `realm` and `authentication_type`) from the current -authenticated user to the current document by pre-processing the ingest. +authenticated user. The `api_key` property exists only if the user authenticates with an API key. It is an object containing the `id`, `name` and `metadata` (if it exists and is non-empty) fields of the API key. diff --git a/docs/reference/mapping/fields/source-field.asciidoc b/docs/reference/mapping/fields/source-field.asciidoc index 903b301ab1a96..38ed697a11708 100644 --- a/docs/reference/mapping/fields/source-field.asciidoc +++ b/docs/reference/mapping/fields/source-field.asciidoc @@ -58,6 +58,9 @@ and <> APIs. automatically. ================================================== +NOTE: You can't disable the `_source` field for indexes with <> +set to `logsdb` or `time_series`. + TIP: If disk space is a concern, rather increase the <> instead of disabling the `_source`. diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 1678441b13bf2..1306872a6308d 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -49,6 +49,8 @@ document. Similarly, malformed values of fields that use <>. +Synthetic source is not supported in <> snapshot repositories. To store indexes that use synthetic `_source`, choose a different repository type. + [[synthetic-source-modifications]] ===== Synthetic `_source` modifications diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc index e66a6852dc159..e78eeddba1878 100644 --- a/docs/reference/mapping/params/doc-values.asciidoc +++ b/docs/reference/mapping/params/doc-values.asciidoc @@ -11,11 +11,13 @@ different data access pattern. Instead of looking up the term and finding documents, we need to be able to look up the document and find the terms that it has in a field. -Doc values are the on-disk data structure, built at document index time, which -makes this data access pattern possible. They store the same values as the -`_source` but in a column-oriented fashion that is way more efficient for -sorting and aggregations. Doc values are supported on almost all field types, -with the __notable exception of `text` and `annotated_text` fields__. +The `doc_values` field is an on-disk data structure that is built at index time and +enables efficient data access. It stores the same values as +`_source`, but in a column-oriented format that is more efficient for +sorting and aggregations. + +Doc values are supported on most field types, +excluding `text` and `annotated_text` fields. See also <>. [[doc-value-only-fields]] ==== Doc-value-only fields @@ -55,11 +57,17 @@ PUT my-index-000001 <2> The `session_id` field has `index` disabled, and is therefore a doc-value-only long field as doc values are enabled by default. +[[doc-values-disable]] ==== Disabling doc values -All fields which support doc values have them enabled by default. If you are -sure that you don't need to sort or aggregate on a field, or access the field -value from a script, you can disable doc values in order to save disk space: +For all fields that support them, `doc_values` are enabled by default. In +some field types, such as <>, +`doc_values` appear in API responses but can't be configured. Setting +`doc_values` for these fields might result in an error or have no effect. + +If you're +certain you don't need to sort or aggregate on a field or access its +value from a script, you can disable `doc_values` in order to save disk space: [source,console] -------------------------------------------------- diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 165d9d7900441..2e397a74a0013 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -74,9 +74,10 @@ The following parameters are accepted by `keyword` fields: <>:: Do not index any string longer than this value. Defaults to `2147483647` - so that all values would be accepted. Please however note that default - dynamic mapping rules create a sub `keyword` field that overrides this - default by setting `ignore_above: 256`. + in standard indices so that all values would be accepted, and `8191` in logsdb + indices to protect against Lucene's term byte-length limit of `32766`. Please + however note that default dynamic mapping rules create a sub `keyword` field + that overrides this default by setting `ignore_above: 256`. <>:: diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index c5cc24f957a44..8ae5e1631f558 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -5,14 +5,12 @@ Semantic text ++++ -beta[] - The `semantic_text` field type automatically generates embeddings for text content using an inference endpoint. Long passages are <> to smaller sections to enable the processing of larger corpuses of text. The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the <>. -This field type and the <> type make it simpler to perform semantic search on your data. +This field type and the <> type make it simpler to perform semantic search on your data. The `semantic_text` field type may also be queried with <>, <> or <> queries. If you don’t specify an inference endpoint, the `inference_id` field defaults to `.elser-2-elasticsearch`, a preconfigured endpoint for the elasticsearch service. @@ -193,8 +191,8 @@ types and create an ingest pipeline with an <> to generate the embeddings. <> walks you through the process. In these cases - when you use `sparse_vector` or `dense_vector` field types instead -of the `semantic_text` field type to customize indexing - using the -<> is not supported for querying the +of the `semantic_text` field type to customize indexing - using the +<> is not supported for querying the field data. diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc index 255e34ecd959b..743ab8c8a055d 100644 --- a/docs/reference/mapping/types/wildcard.asciidoc +++ b/docs/reference/mapping/types/wildcard.asciidoc @@ -123,7 +123,8 @@ The following parameters are accepted by `wildcard` fields: <>:: Do not index any string longer than this value. Defaults to `2147483647` - so that all values would be accepted. + in standard indices so that all values would be accepted, and `8191` in + logsdb indices to protect against Lucene's term byte-length limit of `32766`. [discrete] ==== Limitations diff --git a/docs/reference/migration/apis/data-stream-reindex.asciidoc b/docs/reference/migration/apis/data-stream-reindex.asciidoc index 4641e0fe0911a..ccaed97979727 100644 --- a/docs/reference/migration/apis/data-stream-reindex.asciidoc +++ b/docs/reference/migration/apis/data-stream-reindex.asciidoc @@ -21,9 +21,10 @@ from the original backing indices are copied to the resulting backing indices. This api runs in the background because reindexing all indices in a large data stream is expected to take a large amount of time and resources. The endpoint will return immediately and a persistent task will be created to run in the background. The current status of the task can be checked with -the <>. This status will be available for 24 hours after the task completes, whether -it finished successfully or failed. If the status is still available for a task, the task must be cancelled before it can be re-run. -A running or recently completed data stream reindex task can be cancelled using the <>. +the <>. This status will be available for 24 hours after the task +completes, whether it finished successfully or failed. However, only the last status is retained so re-running a reindex +will overwrite the previous status for that data stream. A running or recently completed data stream reindex task can be +cancelled using the <>. /////////////////////////////////////////////////////////// [source,console] diff --git a/docs/reference/migration/migrate_8_16.asciidoc b/docs/reference/migration/migrate_8_16.asciidoc index ec436599e98b4..94c7a714be4b0 100644 --- a/docs/reference/migration/migrate_8_16.asciidoc +++ b/docs/reference/migration/migrate_8_16.asciidoc @@ -9,6 +9,9 @@ your application to {es} 8.16. See also <> and <>. +coming::[8.16.0] + + [discrete] [[breaking-changes-8.16]] === Breaking changes diff --git a/docs/reference/migration/migrate_8_18.asciidoc b/docs/reference/migration/migrate_8_18.asciidoc index c989ff9f85b6d..d1b9040022523 100644 --- a/docs/reference/migration/migrate_8_18.asciidoc +++ b/docs/reference/migration/migrate_8_18.asciidoc @@ -16,5 +16,124 @@ coming::[8.18.0] [[breaking-changes-8.18]] === Breaking changes -There are no breaking changes in {es} 8.18. +The following changes in {es} 8.18 might affect your applications +and prevent them from operating normally. +Before upgrading to 8.18, review these changes and take the described steps +to mitigate the impact. + +[discrete] +[[breaking_818_analysis_changes]] +==== Analysis changes + +[[change_semantic_text_to_act_like_normal_text_field]] +.Change Semantic Text To Act Like A Normal Text Field +[%collapsible] +==== +*Details* + +The previous semantic_text format used a complex subfield structure in _source to store the embeddings. This complicated interactions/integrations with semantic_text fields and _source in general. This new semantic_text format treats it as a normal text field, where the field's value in _source is the value assigned by the user. + +*Impact* + +Users who parsed the subfield structure of the previous semantic_text format in _source will need to update their parsing logic. The new format does not directly expose the chunks and embeddings generated from the input text. The new format will be applied to all new indices, any existing indices will continue to use the previous format. +==== + +[discrete] +[[breaking_818_cluster_and_node_setting_changes]] +==== Cluster and node setting changes + +[[drop_tls_rsa_cipher_support_for_jdk_24]] +.Drop `TLS_RSA` cipher support for JDK 24 +[%collapsible] +==== +*Details* + +This change removes `TLS_RSA` ciphers from the list of default supported ciphers, for Elasticsearch deployments running on JDK 24. + +*Impact* + +The dropped ciphers are `TLS_RSA_WITH_AES_256_GCM_SHA384`, `TLS_RSA_WITH_AES_128_GCM_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA256`, `TLS_RSA_WITH_AES_128_CBC_SHA256`, `TLS_RSA_WITH_AES_256_CBC_SHA`, and `TLS_RSA_WITH_AES_128_CBC_SHA`. TLS connections to Elasticsearch using these ciphers will no longer work. Please configure your clients to use one of supported cipher suites. +==== + +[discrete] +[[breaking_818_packaging_changes]] +==== Packaging changes + +[[disable_machine_learning_on_macos_x86_64]] +.Disable machine learning on macOS x86_64 +[%collapsible] +==== +*Details* + +The machine learning plugin is permanently disabled on macOS x86_64. For the last three years Apple has been selling hardware based on the arm64 architecture, and support will increasingly focus on this architecture in the future. Changes to upstream dependencies of Elastic's machine learning functionality have made it unviable for Elastic to continue to build machine learning on macOS x86_64. + +*Impact* + +To continue to use machine learning functionality on macOS please switch to an arm64 machine (Apple silicon). Alternatively, it will still be possible to run Elasticsearch with machine learning enabled in a Docker container on macOS x86_64. +==== + +[discrete] +[[breaking_818_rest_api_changes]] +==== REST API changes + +[[set_allow_partial_search_results_true_by_default]] +.Set allow_partial_search_results=true by default +[%collapsible] +==== +*Details* + +Before this change, in case of shard failures, EQL queries always returned an error. With this change, they will keep running and will return partial results. + +*Impact* + +EQL queries that would previously fail due to shard failures, will now succeed and return partial results. The previous defaults can be restored by setting `xpack.eql.default_allow_partial_results` cluster setting to `false` or setting with `allow_partial_search_results` to `false` in the query request. +==== + + +[discrete] +[[deprecated-8.18]] +=== Deprecations + +The following functionality has been deprecated in {es} 8.18 +and will be removed in a future version. +While this won't have an immediate impact on your applications, +we strongly encourage you to take the described steps to update your code +after upgrading to 8.18. + +To find out if you are using any deprecated functionality, +enable <>. + +[discrete] +[[deprecations_818_index_setting]] +==== Index setting deprecations + +[[increase_frozen_indices_deprecation_level_to_critical]] +.Increase the frozen indices deprecation level to `CRITICAL` +[%collapsible] +==== +*Details* + +The migration deprecations API previously returned a warning on frozen indices. Support for reading frozen indices will be removed in 9.0, so in 8.18 this has been made a critical issue. + +*Impact* + +Users are required to unfreeze any frozen indices before upgrading to 9.x. (N.B. It was impossible to freeze indices in 8.x, so this only applies to 7.x indices which have not been reindexed.) +==== + +[discrete] +[[deprecations_818_rest_api]] +==== REST API deprecations + +[[deprecate_ability_to_connect_to_nodes_of_versions_8_17_earlier]] +.Deprecate ability to connect to nodes of versions 8.17 and earlier +[%collapsible] +==== +*Details* + +Versions 9.0.0 and later of {es} will not support communication with nodes of versions earlier than 8.18.0, so the ability to connect to nodes of earlier versions is deprecated in this version. This applies both to communication within a cluster and communication across clusters (e.g. for <> or <>). +{es} will report in its <> each time it opens a connection to a node that will not be supported from version 9.0.0 onwards. You must upgrade all your clusters to version 8.18.0 or later before upgrading any of your clusters to 9.0.0 or later. + +*Impact* + +Upgrade all of your clusters to at least 8.18.0 before upgrading any of them to 9.0.0 or later. +==== + +[[v_7_deprecation_logging_set_to_critical]] +.V_7 deprecation logging set to critical +[%collapsible] +==== +*Details* + +This changes the V_7 API deprecation logging level to CRITICAL. + +*Impact* + +Any usage of deprecated V_7 API features will now be logged at the CRITICAL level. This does not change functionality. +==== diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc index 89eb6e8559056..ec068490912a0 100644 --- a/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc +++ b/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc @@ -231,7 +231,7 @@ The bucket timestamp of the anomaly. The bucket timestamp of the anomaly in ISO8601 format. `context`.`topInfluencers`:: -The list of top influencers. +The list of top influencers. Limited to a maximum of 3 documents. + .Properties of `context.topInfluencers` [%collapsible%open] @@ -248,7 +248,7 @@ influencer's overall contribution to the anomalies. ==== `context`.`topRecords`:: -The list of top records. +The list of top records. Limited to a maximum of 3 documents. + .Properties of `context.topRecords` [%collapsible%open] diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 2ea4dcb9b18f5..50bcea3904e2f 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -292,6 +292,7 @@ include::network/threading.asciidoc[] preview::[] +[[tcp-readiness-port]] If configured, a node can open a TCP port when the node is in a ready state. A node is deemed ready when it has successfully joined a cluster. In a single node configuration, the node is said to be ready, when it's able to accept requests. diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index e8dd995623a1d..dc0c44280c182 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -46,6 +46,29 @@ The following additional roles are available: * `voting_only` +[NOTE] +[[coordinating-only-node]] +.Coordinating node +=============================================== + +Requests like search requests or bulk-indexing requests may involve data held +on different data nodes. A search request, for example, is executed in two +phases which are coordinated by the node which receives the client request -- +the _coordinating node_. + +In the _scatter_ phase, the coordinating node forwards the request to the data +nodes which hold the data. Each data node executes the request locally and +returns its results to the coordinating node. In the _gather_ phase, the +coordinating node reduces each data node's results into a single global +result set. + +Every node is implicitly a coordinating node. This means that a node that has +an explicit empty list of roles in the `node.roles` setting will only act as a coordinating +node, which cannot be disabled. As a result, such a node needs to have enough +memory and CPU in order to deal with the gather phase. + +=============================================== + [IMPORTANT] ==== If you set `node.roles`, ensure you specify every node role your cluster needs. diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc index fc7b6831ca848..b7c0282384e7c 100644 --- a/docs/reference/modules/transport.asciidoc +++ b/docs/reference/modules/transport.asciidoc @@ -195,6 +195,15 @@ setting `transport.ping_schedule` if keepalives cannot be configured. Devices which drop connections when they reach a certain age are a common source of problems to {es} clusters, and must not be used. +If an {es} node is temporarily unable to handle network traffic it may stop +reading data from the network and advertise a zero-length TCP window to its +peers so that they pause the transmission of data to the unavailable node. This +is the standard backpressure mechanism built into TCP. When the node becomes +available again, it will resume reading from the network. Configure your +network to permit TCP connections to exist in this paused state without +disruption. Do not impose any limit on the length of time that a connection may +remain in this paused state. + For information about troubleshooting unexpected network disconnections, see <>. diff --git a/docs/reference/node-roles.asciidoc b/docs/reference/node-roles.asciidoc index e8c1d9143a38e..296c76e6dba9b 100644 --- a/docs/reference/node-roles.asciidoc +++ b/docs/reference/node-roles.asciidoc @@ -359,7 +359,7 @@ node.roles: [ ingest ] ---- [discrete] -[[coordinating-only-node]] +[[coordinating-only-node-role]] ==== Coordinating only node If you take away the ability to be able to handle master duties, to hold data, diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index 27220f0d85149..efd37a4dc2065 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -12,28 +12,21 @@ occurrence types are: [cols="<,<",options="header",] |======================================================================= |Occur |Description -|`must` |The clause (query) must appear in matching documents and will -contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match _all_ the specified queries. +|`must` |The clause (query) must appear in matching documents and will contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match _all_ the specified queries. |`should` |The clause (query) should appear in the matching document. Each query defined under a `should` acts as a logical "OR", returning documents that match _any_ of the specified queries. |`filter` |The clause (query) must appear in matching documents. However unlike -`must` the score of the query will be ignored. Filter clauses are executed -in <>, meaning that scoring is ignored -and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match _all_ the specified queries. +`must` the score of the query will be ignored. Filter clauses are executed in <>, meaning that scoring is ignored and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match _all_ the specified queries. |`must_not` |The clause (query) must not appear in the matching -documents. Clauses are executed in <> meaning -that scoring is ignored and clauses are considered for caching. Because scoring is -ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. +documents. Clauses are executed in <> meaning that scoring is ignored and clauses are considered for caching. Because scoring is ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. |======================================================================= -The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results will not be cached for faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query. +The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results are not cached, which means repeated queries won't benefit from faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query. -The `bool` query takes a _more-matches-is-better_ approach, so the score from -each matching `must` or `should` clause will be added together to provide the -final `_score` for each document. +The `bool` query takes a _more-matches-is-better_ approach, so the score from each matching `must` or `should` clause will be added together to provide the final `_score` for each document. [source,console] -------------------------------------------------- @@ -76,6 +69,56 @@ Otherwise, the default value is `0`. For other valid values, see the <>. +[[nested-bool-queries]] +==== Nested bool queries + +You can nest `bool` queries within other `bool` queries to create complex logical constructs. This allows you to build sophisticated search conditions by combining multiple levels of boolean logic. + +For example: + +[source,console] +---- +GET /_search +{ + "query": { + "bool": { + "must": [ <1> + { + "bool": { + "should": [ + { "match": { "user.id": "kimchy" }}, + { "match": { "user.id": "banon" }} + ] + } + }, + { "match": { "tags": "production" }} + ], + "should": [ <2> + { + "bool": { + "must": [ + { "match": { "status": "active" }}, + { "match": { "title": "quick brown fox" }} + ] + } + } + ] + } + } +} +---- + +<1> Only documents that match all conditions in the must section will be returned in the results. This means documents must match either "kimchy" OR "banon" in the user.id field AND "production" in the tags field. It is semantically equivalent to (user.id="kimchy" OR user.id="banon") AND tags="production" + +<2> Matches in the `should` clauses are optional. They will only boost the relevance scores of documents that already match the required `must` criteria and don't add new documents to the result set. It is semantically equivalent to (status="active" AND title="quick brown fox") + +You can use the `minimum_should_match` parameter to require matches from the `should` clauses. + +[NOTE] +==== +While nesting `bool` queries can be powerful, it can also lead to complex and slow queries. Try to keep your queries as flat as possible for the best performance. +==== + [[score-bool-filter]] ==== Scoring with `bool.filter` diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 25b995eefc219..493d90cea93bd 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -133,7 +133,6 @@ The following parameters are required when percolating a document: This is an optional parameter. `document`:: The source of the document being percolated. `documents`:: Like the `document` parameter, but accepts multiple documents via a json array. -`document_type`:: The type / mapping of the document being percolated. This parameter is deprecated and will be removed in Elasticsearch 8.0. Instead of specifying the source of the document being percolated, the source can also be retrieved from an already stored document. The `percolate` query will then internally execute a get request to fetch that document. @@ -142,7 +141,6 @@ In that case the `document` parameter can be substituted with the following para [horizontal] `index`:: The index the document resides in. This is a required parameter. -`type`:: The type of the document to fetch. This parameter is deprecated and will be removed in Elasticsearch 8.0. `id`:: The id of the document to fetch. This is a required parameter. `routing`:: Optionally, routing to be used to fetch document to percolate. `preference`:: Optionally, preference to be used to fetch document to percolate. diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc index 914f4429f7f9c..d3562bf413063 100644 --- a/docs/reference/query-dsl/semantic-query.asciidoc +++ b/docs/reference/query-dsl/semantic-query.asciidoc @@ -4,8 +4,6 @@ Semantic ++++ -beta[] - The `semantic` query type enables you to perform <> on data stored in a <> field. diff --git a/docs/reference/quickstart/esql-search-tutorial.asciidoc b/docs/reference/quickstart/esql-search-tutorial.asciidoc new file mode 100644 index 0000000000000..fad6668db9f77 --- /dev/null +++ b/docs/reference/quickstart/esql-search-tutorial.asciidoc @@ -0,0 +1,485 @@ +// ℹ️ 9.x version of this doc lives in docs-content repo +// https://github.com/elastic/docs-content/blob/main/solutions/search/esql-search-tutorial.md + +[[esql-search-tutorial]] +== Tutorial: Search and filter with {esql} + +[TIP] +===== +This tutorial presents examples in {esql} syntax. Refer to <> for the equivalent examples in Query DSL syntax. +===== + +This is a hands-on introduction to the basics of full-text search and semantic search, using <>. + +For an overview of all the search capabilities in {esql}, refer to <>. + +In this scenario, we're implementing search for a cooking blog. The blog contains recipes with various attributes including textual content, categorical data, and numerical ratings. + +[discrete] +[[esql-search-tutorial-requirements]] +=== Requirements + +You'll need a running {es} cluster, together with {kib} to use the Dev Tools API Console. Refer to <> for deployment options. + +Want to get started quickly? Run the following command in your terminal to set up a <>: + +[source,sh] +---- +curl -fsSL https://elastic.co/start-local | sh +---- +// NOTCONSOLE + +[discrete] +[[esql-search-tutorial-running-esql-queries]] +=== Running {esql} queries + +In this tutorial, you'll see {esql} examples in the following format: + +[source,esql] +---- +FROM cooking_blog +| WHERE description:"fluffy pancakes" +| LIMIT 1000 +---- + +If you want to run these queries in the <>, you'll need to use the following syntax: + +[source,js] +---- +POST /_query?format=txt +{ + "query": """ + FROM cooking_blog + | WHERE description:"fluffy pancakes" + | LIMIT 1000 + """ +} +---- +// NOTCONSOLE + +If you'd prefer to use your favorite programming language, refer to <> for a list of official and community-supported clients. + +[discrete] +[[esql-search-tutorial-step-1-create-an-index]] +=== Step 1: Create an index + +Create the `cooking_blog` index to get started: + +[source,console] +---- +PUT /cooking_blog +---- +// TESTSETUP + +Now define the mappings for the index: + +[source,console] +---- +PUT /cooking_blog/_mapping +{ + "properties": { + "title": { + "type": "text", + "analyzer": "standard", <1> + "fields": { <2> + "keyword": { + "type": "keyword", + "ignore_above": 256 <3> + } + } + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "author": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "date": { + "type": "date", + "format": "yyyy-MM-dd" + }, + "category": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "rating": { + "type": "float" + } + } +} +---- +// TEST + +<1> The `standard` analyzer is used by default for `text` fields if an `analyzer` isn't specified. It's included here for demonstration purposes. +<2> <> are used here to index `text` fields as both `text` and `keyword` <>. This enables both full-text search and exact matching/filtering on the same field. Note that if you used <>, these multi-fields would be created automatically. +<3> The <> prevents indexing values longer than 256 characters in the `keyword` field. Again this is the default value, but it's included here for demonstration purposes. It helps to save disk space and avoid potential issues with Lucene's term byte-length limit. + +[TIP] +===== +Full-text search is powered by <>. Text analysis normalizes and standardizes text data so it can be efficiently stored in an inverted index and searched in near real-time. Analysis happens at both <>. This tutorial won't cover analysis in detail, but it's important to understand how text is processed to create effective search queries. +===== + +[discrete] +[[esql-search-tutorial-index-data]] +=== Step 2: Add sample blog posts to your index + +Now you'll need to index some example blog posts using the https://www.elastic.co/docs/api/doc/elasticsearch/v8/operation/operation-indices-get-mapping[Bulk API]. Note that `text` fields are analyzed and multi-fields are generated at index time. + +[source,console] +---- +POST /cooking_blog/_bulk?refresh=wait_for +{"index":{"_id":"1"}} +{"title":"Perfect Pancakes: A Fluffy Breakfast Delight","description":"Learn the secrets to making the fluffiest pancakes, so amazing you won't believe your tastebuds. This recipe uses buttermilk and a special folding technique to create light, airy pancakes that are perfect for lazy Sunday mornings.","author":"Maria Rodriguez","date":"2023-05-01","category":"Breakfast","tags":["pancakes","breakfast","easy recipes"],"rating":4.8} +{"index":{"_id":"2"}} +{"title":"Spicy Thai Green Curry: A Vegetarian Adventure","description":"Dive into the flavors of Thailand with this vibrant green curry. Packed with vegetables and aromatic herbs, this dish is both healthy and satisfying. Don't worry about the heat - you can easily adjust the spice level to your liking.","author":"Liam Chen","date":"2023-05-05","category":"Main Course","tags":["thai","vegetarian","curry","spicy"],"rating":4.6} +{"index":{"_id":"3"}} +{"title":"Classic Beef Stroganoff: A Creamy Comfort Food","description":"Indulge in this rich and creamy beef stroganoff. Tender strips of beef in a savory mushroom sauce, served over a bed of egg noodles. It's the ultimate comfort food for chilly evenings.","author":"Emma Watson","date":"2023-05-10","category":"Main Course","tags":["beef","pasta","comfort food"],"rating":4.7} +{"index":{"_id":"4"}} +{"title":"Vegan Chocolate Avocado Mousse","description":"Discover the magic of avocado in this rich, vegan chocolate mousse. Creamy, indulgent, and secretly healthy, it's the perfect guilt-free dessert for chocolate lovers.","author":"Alex Green","date":"2023-05-15","category":"Dessert","tags":["vegan","chocolate","avocado","healthy dessert"],"rating":4.5} +{"index":{"_id":"5"}} +{"title":"Crispy Oven-Fried Chicken","description":"Get that perfect crunch without the deep fryer! This oven-fried chicken recipe delivers crispy, juicy results every time. A healthier take on the classic comfort food.","author":"Maria Rodriguez","date":"2023-05-20","category":"Main Course","tags":["chicken","oven-fried","healthy"],"rating":4.9} +---- + +[[step-3-perform-basic-full-text-searches]] +[discrete] +=== Step 3: Perform basic full-text searches + +Full-text search involves executing text-based queries across one or more document fields. These queries calculate a relevance score for each matching document, based on how closely the document's content aligns with the search terms. Elasticsearch offers various query types, each with its own method for matching text and relevance scoring. + +[TIP] +===== +{esql} provides two ways to perform full-text searches: + +1. Full <> syntax: `match(field, "search terms")` +2. Compact syntax using the <>: `field:"search terms"` + +Both are equivalent and can be used interchangeably. The compact syntax is more concise, while the function syntax allows for more configuration options. We'll use the compact syntax in most examples for brevity. + +Refer to the <> reference docs for advanced parameters available with the function syntax. +===== + +[discrete] +[[esql-search-tutorial-basic-full-text-query]] +==== Basic full-text query + +Here's how to search the `description` field for "fluffy pancakes": + +[source,esql] +---- +FROM cooking_blog <1> +| WHERE description:"fluffy pancakes" <2> +| LIMIT 1000 <3> +---- +<1> Specify the index to search +<2> Full-text search with OR logic by default +<3> Return up to 1000 results + +[NOTE] +===== +The results ordering isn't by relevance, as we haven't requested the `_score` metadata field. We'll cover relevance scoring in the next section. +===== + +By default, like the Query DSL `match` query, {esql} uses `OR` logic between terms. This means it will match documents that contain either "fluffy" or "pancakes", or both, in the description field. + +[TIP] +===== +You can control which fields to include in the response using the `KEEP` command: + +[source,esql] +---- +FROM cooking_blog +| WHERE description:"fluffy pancakes" +| KEEP title, description, rating <1> +| LIMIT 1000 +---- +<1> Select only specific fields to include in response +===== + +[discrete] +[[esql-search-tutorial-require-all-terms]] +==== Require all terms in a match query + +Sometimes you need to require that all search terms appear in the matching documents. Here's how to do that using the function syntax with the `operator` parameter: + +[source,esql] +---- +FROM cooking_blog +| WHERE match(description, "fluffy pancakes", {"operator": "AND"}) <1> +| LIMIT 1000 +---- +<1> Require ALL terms to match + +This stricter search returns *zero hits* on our sample data, as no document contains both "fluffy" and "pancakes" in the description. + +[discrete] +[[esql-search-tutorial-minimum-terms]] +==== Specify a minimum number of terms to match + +Sometimes requiring all terms is too strict, but the default OR behavior is too lenient. You can specify a minimum number of terms that must match: + +[source,esql] +---- +FROM cooking_blog +| WHERE match(title, "fluffy pancakes breakfast", {"minimum_should_match": 2}) +| LIMIT 1000 +---- + +This query searches the title field to match at least 2 of the 3 terms: "fluffy", "pancakes", or "breakfast". + +[discrete] +[[esql-search-tutorial-semantic-search]] +=== Step 4: Semantic search and hybrid search + +[discrete] +[[esql-search-tutorial-index-semantic-content]] +==== Index semantic content + +{es} allows you to semantically search for documents based on the meaning of the text, rather than just the presence of specific keywords. This is useful when you want to find documents that are conceptually similar to a given query, even if they don't contain the exact search terms. + +ES|QL supports semantic search when your mappings include fields of the <> type. This example mapping update adds a new field called `semantic_description` with the type `semantic_text`: + +[source,console] +---- +PUT /cooking_blog/_mapping +{ + "properties": { + "semantic_description": { + "type": "semantic_text" + } + } +} +---- + +Next, index a document with content into the new field: + +[source,console] +---- +POST /cooking_blog/_doc +{ + "title": "Mediterranean Quinoa Bowl", + "semantic_description": "A protein-rich bowl with quinoa, chickpeas, fresh vegetables, and herbs. This nutritious Mediterranean-inspired dish is easy to prepare and perfect for a quick, healthy dinner.", + "author": "Jamie Oliver", + "date": "2023-06-01", + "category": "Main Course", + "tags": ["vegetarian", "healthy", "mediterranean", "quinoa"], + "rating": 4.7 +} +---- +// TEST[skip:uses ML] + +[discrete] +[[esql-search-tutorial-perform-semantic-search]] +==== Perform semantic search + +Once the document has been processed by the underlying model running on the inference endpoint, you can perform semantic searches. Here's an example natural language query against the `semantic_description` field: + +[source,esql] +---- +FROM cooking_blog +| WHERE semantic_description:"What are some easy to prepare but nutritious plant-based meals?" +| LIMIT 5 +---- + +[TIP] +===== +Follow this <> if you'd like to test out the semantic search workflow against a large dataset. +===== + +[discrete] +[[esql-search-tutorial-perform-hybrid-search]] +==== Perform hybrid search + +You can combine full-text and semantic queries. In this example we combine full-text and semantic search with custom weights: + +[source,esql] +---- +FROM cooking_blog METADATA _score +| WHERE match(semantic_description, "easy to prepare vegetarian meals", { "boost": 0.75 }) + OR match(tags, "vegetarian", { "boost": 0.25 }) +| SORT _score DESC +| LIMIT 5 +---- + +[discrete] +[[esql-search-tutorial-search-across-fields]] +=== Step 5: Search across multiple fields at once + +When users enter a search query, they often don't know (or care) whether their search terms appear in a specific field. {esql} provides ways to search across multiple fields simultaneously: + +[source,esql] +---- +FROM cooking_blog +| WHERE title:"vegetarian curry" OR description:"vegetarian curry" OR tags:"vegetarian curry" +| LIMIT 1000 +---- + +This query searches for "vegetarian curry" across the title, description, and tags fields. Each field is treated with equal importance. + +However, in many cases, matches in certain fields (like the title) might be more relevant than others. We can adjust the importance of each field using scoring: + +[source,esql] +---- +FROM cooking_blog METADATA _score <1> +| WHERE match(title, "vegetarian curry", {"boost": 2.0}) <2> + OR match(description, "vegetarian curry") + OR match(tags, "vegetarian curry") +| KEEP title, description, tags, _score <3> +| SORT _score DESC <4> +| LIMIT 1000 +---- +<1> Request _score metadata for relevance-based results +<2> Title matches are twice as important +<3> Include relevance score in results +<4> You must explicitly sort by `_score` to see relevance-based results + +[TIP] +===== +When working with relevance scoring in ES|QL, it's important to understand `_score`. If you don't include `METADATA _score` in your query, you won't see relevance scores in your results. This means you won't be able to sort by relevance or filter based on relevance scores. + +When you include `METADATA _score`, search functions included in WHERE conditions contribute to the relevance score. Filtering operations (like range conditions and exact matches) don't affect the score. + +If you want the most relevant results first, you must sort by `_score`, by explicitly using `SORT _score DESC` or `SORT _score ASC`. +===== + +[discrete] +[[esql-search-tutorial-filter-exact-matches]] +=== Step 6: Filter and find exact matches + +Filtering allows you to narrow down your search results based on exact criteria. Unlike full-text searches, filters are binary (yes/no) and do not affect the relevance score. Filters execute faster than queries because excluded results don't need to be scored. + +[source,esql] +---- +FROM cooking_blog +| WHERE category.keyword == "Breakfast" <1> +| KEEP title, author, rating, tags +| SORT rating DESC +| LIMIT 1000 +---- +<1> Exact match using keyword field (case-sensitive) + +Note the use of `category.keyword` here. This refers to the <> multi-field of the `category` field, ensuring an exact, case-sensitive match. + +[discrete] +[[esql-search-tutorial-date-range]] +==== Search for posts within a date range + +Often users want to find content published within a specific time frame: + +[source,esql] +---- +FROM cooking_blog +| WHERE date >= "2023-05-01" AND date <= "2023-05-31" <1> +| KEEP title, author, date, rating +| LIMIT 1000 +---- +<1> Inclusive date range filter + +[discrete] +[[esql-search-tutorial-exact-matches]] +==== Find exact matches + +Sometimes users want to search for exact terms to eliminate ambiguity in their search results: + +[source,esql] +---- +FROM cooking_blog +| WHERE author.keyword == "Maria Rodriguez" <1> +| KEEP title, author, rating, tags +| SORT rating DESC +| LIMIT 1000 +---- +<1> Exact match on author + +Like the `term` query in Query DSL, this has zero flexibility and is case-sensitive. + +[discrete] +[[esql-search-tutorial-combine-criteria]] +=== Step 7: Combine multiple search criteria + +Complex searches often require combining multiple search criteria: + +[source,esql] +---- +FROM cooking_blog METADATA _score +| WHERE rating >= 4.5 <1> + AND NOT category.keyword == "Dessert" <2> + AND (title:"curry spicy" OR description:"curry spicy") <3> +| SORT _score DESC +| KEEP title, author, rating, tags, description +| LIMIT 1000 +---- +<1> Numerical filter +<2> Exclusion filter +<3> Full-text search in multiple fields + +[discrete] +[[esql-search-tutorial-relevance-scoring]] +==== Combine relevance scoring with custom criteria + +For more complex relevance scoring with combined criteria, you can use the `EVAL` command to calculate custom scores: + +[source,esql] +---- +FROM cooking_blog METADATA _score +| WHERE NOT category.keyword == "Dessert" +| EVAL tags_concat = MV_CONCAT(tags.keyword, ",") <1> +| WHERE tags_concat LIKE "*vegetarian*" AND rating >= 4.5 <2> +| WHERE match(title, "curry spicy", {"boost": 2.0}) OR match(description, "curry spicy") <3> +| EVAL category_boost = CASE(category.keyword == "Main Course", 1.0, 0.0) <4> +| EVAL date_boost = CASE(DATE_DIFF("month", date, NOW()) <= 1, 0.5, 0.0) <5> +| EVAL custom_score = _score + category_boost + date_boost <6> +| WHERE custom_score > 0 <7> +| SORT custom_score DESC +| LIMIT 1000 +---- +<1> Convert multi-value field to string +<2> Wildcard pattern matching +<3> Uses full text functions, will update _score metadata field +<4> Conditional boost +<5> Boost recent content +<6> Combine scores +<7> Filter based on custom score + +[discrete] +[[esql-search-tutorial-learn-more]] +=== Learn more + +[discrete] +[[esql-search-tutorial-documentation]] +==== Documentation + +This tutorial introduced the basics of search and filtering in {esql}. Building a real-world search experience requires understanding many more advanced concepts and techniques. Here are some resources once you're ready to dive deeper: + +- <>: Learn about all your options for search use cases with {esql}. +- <>: Explore the full list of search functions available in {esql}. +- <>: Understand your various options for semantic search in Elasticsearch. + - <>: Learn how to use the `semantic_text` field type for semantic search. This is the recommended approach for most users looking to perform semantic search in {es}, because it abstracts away the complexity of setting up inference endpoints and models. + +[discrete] +[[esql-search-tutorial-blog-posts]] +==== Related blog posts + +// TODO [[uncomment once blog is live]] - https://www.elastic.co/blog/esql-you-know-for-search-scoring-semantic-search[Introducing scoring and semantic searchin {esql}]: +- https://www.elastic.co/search-labs/blog/filtering-in-esql-full-text-search-match-qstr[Introducing full text filtering in ES|QL] \ No newline at end of file diff --git a/docs/reference/quickstart/index.asciidoc b/docs/reference/quickstart/index.asciidoc index 330582956c457..ccb7968accc0f 100644 --- a/docs/reference/quickstart/index.asciidoc +++ b/docs/reference/quickstart/index.asciidoc @@ -25,7 +25,8 @@ Alternatively, refer to our <>. Learn about indices, documents, and mappings, and perform a basic search using the Query DSL. * <>. Learn about different options for querying data, including full-text search and filtering, using the Query DSL. -* <>: Learn how to query and aggregate your data using {esql}. +* <>: Learn how to query and aggregate your data using {esql}. +* <>: Learn how to use {esql} for search use cases, including full-text search, semantic search, and hybrid search. * <>. Learn how to analyze data using different types of aggregations, including metrics, buckets, and pipelines. * <>: Learn how to create embeddings for your data with `semantic_text` and query using the `semantic` query. ** <>: Learn how to combine semantic search with full-text search. @@ -42,4 +43,5 @@ If you're interested in using {es} with Python, check out Elastic Search Labs: include::getting-started.asciidoc[] include::full-text-filtering-tutorial.asciidoc[] +include::esql-search-tutorial.asciidoc[] include::aggs-tutorial.asciidoc[] diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 97993317c0402..32bed8c27218f 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -7,8 +7,14 @@ This section summarizes the changes in each release. * <> +* <> +* <> +* <> * <> * <> +* <> +* <> +* <> * <> * <> * <> @@ -83,8 +89,14 @@ This section summarizes the changes in each release. -- include::release-notes/8.18.0.asciidoc[] +include::release-notes/8.17.4.asciidoc[] +include::release-notes/8.17.3.asciidoc[] +include::release-notes/8.17.2.asciidoc[] include::release-notes/8.17.1.asciidoc[] include::release-notes/8.17.0.asciidoc[] +include::release-notes/8.16.6.asciidoc[] +include::release-notes/8.16.5.asciidoc[] +include::release-notes/8.16.4.asciidoc[] include::release-notes/8.16.3.asciidoc[] include::release-notes/8.16.2.asciidoc[] include::release-notes/8.16.1.asciidoc[] diff --git a/docs/reference/release-notes/8.16.4.asciidoc b/docs/reference/release-notes/8.16.4.asciidoc new file mode 100644 index 0000000000000..47fff79f1c3ec --- /dev/null +++ b/docs/reference/release-notes/8.16.4.asciidoc @@ -0,0 +1,42 @@ +[[release-notes-8.16.4]] +== {es} version 8.16.4 + +Also see <>. + +[[bug-8.16.4]] +[float] +=== Bug fixes + +Aggregations:: +* Aggregations cancellation after collection {es-pull}120944[#120944] (issue: {es-issue}108701[#108701]) + +Data streams:: +* Map `scope.name` as a dimension {es-pull}120590[#120590] + +ES|QL:: +* Use `field_caps` native nested fields filtering {es-pull}117201[#117201] (issue: {es-issue}117054[#117054]) + +Mapping:: +* Fix synthetic source issue with deeply nested ignored source fields {es-pull}121715[#121715] + +Ranking:: +* Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled {es-pull}120717[#120717] (issue: {es-issue}120739[#120739]) +* LTR sometines throw `NullPointerException:` Cannot read field "approximation" because "top" is null {es-pull}120809[#120809] +* Normalize negative scores for `text_similarity_reranker` retriever {es-pull}120930[#120930] (issue: {es-issue}120201[#120201]) +* Update Text Similarity Reranker to Properly Handle Aliases {es-pull}120062[#120062] (issue: {es-issue}119617[#119617]) + +Search:: +* Fix potential file leak in ES816BinaryQuantizedVectorsWriter {es-pull}120014[#120014] (issue: {es-issue}119981[#119981]) + +Snapshot/Restore:: +* Cheaper snapshot-related `toString()` impls {es-pull}121283[#121283] +* Issue S3 web identity token refresh call with sufficient permissions {es-pull}119748[#119748] (issue: {es-issue}119747[#119747]) + +[[enhancement-8.16.4]] +[float] +=== Enhancements + +Ingest Node:: +* Improve memory aspects of enrich cache {es-pull}120256[#120256] (issues: {es-issue}96050[#96050], {es-issue}120021[#120021]) + + diff --git a/docs/reference/release-notes/8.16.5.asciidoc b/docs/reference/release-notes/8.16.5.asciidoc new file mode 100644 index 0000000000000..6d65295af9305 --- /dev/null +++ b/docs/reference/release-notes/8.16.5.asciidoc @@ -0,0 +1,47 @@ +[[release-notes-8.16.5]] +== {es} version 8.16.5 + +Also see <>. + +[[bug-8.16.5]] +[float] +=== Bug fixes + +Allocation:: +* Deduplicate allocation stats calls {es-pull}123246[#123246] + +Authentication:: +* Improve jwt logging on failed auth {es-pull}122247[#122247] + +CRUD:: +* Reduce license checks in `LicensedWriteLoadForecaster` {es-pull}123346[#123346] (issue: {es-issue}123247[#123247]) + +Data streams:: +* Add `_metric_names_hash` field to OTel metric mappings {es-pull}120952[#120952] + +EQL:: +* Fix JOIN command validation (not supported) {es-pull}122011[#122011] + +ES|QL:: +* Fix ENRICH validation for use of wildcards {es-pull}121911[#121911] +* Speed up VALUES for many buckets {es-pull}123073[#123073] + +Ingest:: +* Fix `ArrayIndexOutOfBoundsException` in `ShardBulkInferenceActionFilter` {es-pull}122538[#122538] + +Ingest Node:: +* Canonicalize processor names and types in `IngestStats` {es-pull}122610[#122610] +* Deduplicate `IngestStats` and `IngestStats.Stats` identity records when deserializing {es-pull}122496[#122496] +* Fix redact processor arraycopy bug {es-pull}122640[#122640] +* Register `IngestGeoIpMetadata` as a NamedXContent {es-pull}123079[#123079] + +Mapping:: +* fix stale data in synthetic source for string stored field {es-pull}123105[#123105] (issue: {es-issue}123110[#123110]) + +[[upgrade-8.16.5]] +[float] +=== Upgrades + +Authentication:: +* Bump json-smart and oauth2-oidc-sdk {es-pull}122737[#122737] + diff --git a/docs/reference/release-notes/8.16.6.asciidoc b/docs/reference/release-notes/8.16.6.asciidoc new file mode 100644 index 0000000000000..c4d4b085006c8 --- /dev/null +++ b/docs/reference/release-notes/8.16.6.asciidoc @@ -0,0 +1,26 @@ +[[release-notes-8.16.6]] +== {es} version 8.16.6 + +Also see <>. + +[[bug-8.16.6]] +[float] +=== Bug fixes + +Infra/Core:: +* Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. {es-pull}124732[#124732] (issue: {es-issue}124667[#124667]) + +Machine Learning:: +* Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints {es-pull}124769[#124769] (issue: {es-issue}124675[#124675]) + +Search:: +* Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions {es-pull}123890[#123890] +* Revert fail-fast disconnect strategy for `_resolve/cluster` {es-pull}124241[#124241] + +[[upgrade-8.16.6]] +[float] +=== Upgrades + +Security:: +* Bump nimbus-jose-jwt to 10.0.2 {es-pull}124544[#124544] + diff --git a/docs/reference/release-notes/8.17.2.asciidoc b/docs/reference/release-notes/8.17.2.asciidoc new file mode 100644 index 0000000000000..c5ccc0cb7d4dd --- /dev/null +++ b/docs/reference/release-notes/8.17.2.asciidoc @@ -0,0 +1,51 @@ +[[release-notes-8.17.2]] +== {es} version 8.17.2 + +Also see <>. + +[[bug-8.17.2]] +[float] +=== Bug fixes + +Authentication:: +* Fix NPE on disabled API auth key cache {es-pull}120483[#120483] + +Data streams:: +* Map `scope.name` as a dimension {es-pull}120590[#120590] + +ES|QL:: +* Use `field_caps` native nested fields filtering {es-pull}117201[#117201] (issue: {es-issue}117054[#117054]) + +Machine Learning:: +* Fix infer on and elasticsearch service endpoint created with a deployment id {es-pull}121428[#121428] +* Fix inference update API calls with `task_type` in body or `deployment_id` defined {es-pull}121231[#121231] +* Skip Usage stats update when ML is disabled {es-pull}121559[#121559] (issue: {es-issue}121532[#121532]) + +Mapping:: +* Fix synthetic source issue with deeply nested ignored source fields {es-pull}121715[#121715] +* Lower `_source.mode` mapping attribute deprecation issue level {es-pull}120059[#120059] + +Ranking:: +* Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled {es-pull}120717[#120717] (issue: {es-issue}120739[#120739]) +* LTR sometines throw `NullPointerException:` Cannot read field "approximation" because "top" is null {es-pull}120809[#120809] +* Normalize negative scores for `text_similarity_reranker` retriever {es-pull}120930[#120930] (issue: {es-issue}120201[#120201]) +* Update Text Similarity Reranker to Properly Handle Aliases {es-pull}120062[#120062] (issue: {es-issue}119617[#119617]) + +Search:: +* Add back `keep_alive` to `async_search.submit` rest-api-spec {es-pull}120781[#120781] +* Fix NPE caused by race condition in async search when minimise round trips is true {es-pull}117504[#117504] +* Fix potential file leak in ES816BinaryQuantizedVectorsWriter {es-pull}120014[#120014] (issue: {es-issue}119981[#119981]) +* Use approximation to advance matched queries {es-pull}120133[#120133] (issue: {es-issue}120130[#120130]) + +Snapshot/Restore:: +* Cheaper snapshot-related `toString()` impls {es-pull}121283[#121283] +* Issue S3 web identity token refresh call with sufficient permissions {es-pull}119748[#119748] (issue: {es-issue}119747[#119747]) + +[[enhancement-8.17.2]] +[float] +=== Enhancements + +Ingest Node:: +* Improve memory aspects of enrich cache {es-pull}120256[#120256] (issues: {es-issue}96050[#96050], {es-issue}120021[#120021]) + + diff --git a/docs/reference/release-notes/8.17.3.asciidoc b/docs/reference/release-notes/8.17.3.asciidoc new file mode 100644 index 0000000000000..784edae5d208b --- /dev/null +++ b/docs/reference/release-notes/8.17.3.asciidoc @@ -0,0 +1,63 @@ +[[release-notes-8.17.3]] +== {es} version 8.17.3 + +Also see <>. + +[[bug-8.17.3]] +[float] +=== Bug fixes + +Aggregations:: +* Disable concurrency when `top_hits` sorts on anything but `_score` {es-pull}123610[#123610] + +Allocation:: +* Deduplicate allocation stats calls {es-pull}123246[#123246] + +Authentication:: +* Improve jwt logging on failed auth {es-pull}122247[#122247] + +CRUD:: +* Reduce license checks in `LicensedWriteLoadForecaster` {es-pull}123346[#123346] (issue: {es-issue}123247[#123247]) + +Data streams:: +* Add `_metric_names_hash` field to OTel metric mappings {es-pull}120952[#120952] + +EQL:: +* Fix JOIN command validation (not supported) {es-pull}122011[#122011] + +ES|QL:: +* Fix ENRICH validation for use of wildcards {es-pull}121911[#121911] +* Fix listener leak in exchange service {es-pull}122417[#122417] (issue: {es-issue}122271[#122271]) +* Speed up VALUES for many buckets {es-pull}123073[#123073] + +Infra/Node Lifecycle:: +* Block running ES 8.17 with JDK 24+ {es-pull}122517[#122517] + +Ingest:: +* Fix `ArrayIndexOutOfBoundsException` in `ShardBulkInferenceActionFilter` {es-pull}122538[#122538] + +Ingest Node:: +* Canonicalize processor names and types in `IngestStats` {es-pull}122610[#122610] +* Deduplicate `IngestStats` and `IngestStats.Stats` identity records when deserializing {es-pull}122496[#122496] +* Fix redact processor arraycopy bug {es-pull}122640[#122640] +* Register `IngestGeoIpMetadata` as a NamedXContent {es-pull}123079[#123079] +* Use ordered maps for `PipelineConfiguration` xcontent deserialization {es-pull}123403[#123403] + +Logs:: +* Fix issues that prevents using search only snapshots for indices that use index sorting. This is includes Logsdb and time series indices. {es-pull}122199[#122199] +* Use min node version to guard injecting settings in logs provider {es-pull}123005[#123005] (issue: {es-issue}122950[#122950]) + +Mapping:: +* Fix synthetic source bug that would mishandle nested `dense_vector` fields {es-pull}122425[#122425] +* fix stale data in synthetic source for string stored field {es-pull}123105[#123105] (issue: {es-issue}123110[#123110]) + +Stats:: +* Fixing serialization of `ScriptStats` `cache_evictions_history` {es-pull}123384[#123384] + +[[upgrade-8.17.3]] +[float] +=== Upgrades + +Authentication:: +* Bump json-smart and oauth2-oidc-sdk {es-pull}122737[#122737] + diff --git a/docs/reference/release-notes/8.17.4.asciidoc b/docs/reference/release-notes/8.17.4.asciidoc new file mode 100644 index 0000000000000..b4f1864449d6c --- /dev/null +++ b/docs/reference/release-notes/8.17.4.asciidoc @@ -0,0 +1,36 @@ +[[release-notes-8.17.4]] +== {es} version 8.17.4 + +Also see <>. + +[[bug-8.17.4]] +[float] +=== Bug fixes + +ES|QL:: +* Catch parsing exception {es-pull}124958[#124958] (issue: {es-issue}119025[#119025]) +* Fix early termination in `LuceneSourceOperator` {es-pull}123197[#123197] + +Indices APIs:: +* Avoid hoarding cluster state references during rollover {es-pull}124107[#124107] (issue: {es-issue}123893[#123893]) +* [8.17] Avoid hoarding cluster state references during rollover {es-pull}124267[#124267] + +Infra/Core:: +* Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size. {es-pull}124732[#124732] (issue: {es-issue}124667[#124667]) + +Machine Learning:: +* Migrate `model_version` to `model_id` when parsing persistent elser inference endpoints {es-pull}124769[#124769] (issue: {es-issue}124675[#124675]) + +Search:: +* Do not let `ShardBulkInferenceActionFilter` unwrap / rewrap ESExceptions {es-pull}123890[#123890] +* Don't generate stacktrace in `TaskCancelledException` {es-pull}125002[#125002] +* Fix concurrency issue in `ScriptSortBuilder` {es-pull}123757[#123757] +* Revert fail-fast disconnect strategy for `_resolve/cluster` {es-pull}124241[#124241] + +[[upgrade-8.17.4]] +[float] +=== Upgrades + +Security:: +* Bump nimbus-jose-jwt to 10.0.2 {es-pull}124544[#124544] + diff --git a/docs/reference/release-notes/8.18.0.asciidoc b/docs/reference/release-notes/8.18.0.asciidoc index 332edfbc23eb7..71ec9efe3104e 100644 --- a/docs/reference/release-notes/8.18.0.asciidoc +++ b/docs/reference/release-notes/8.18.0.asciidoc @@ -5,4 +5,449 @@ coming[8.18.0] Also see <>. +[[breaking-8.18.0]] +[float] +=== Breaking changes + +EQL:: +* Set allow_partial_search_results=true by default {es-pull}120267[#120267] + +Machine Learning:: +* Disable machine learning on macOS x86_64 {es-pull}104125[#104125] + +Search:: +* Change Semantic Text To Act Like A Normal Text Field {es-pull}120813[#120813] + +TLS:: +* Drop `TLS_RSA` cipher support for JDK 24 {es-pull}123600[#123600] + +[[bug-8.18.0]] +[float] +=== Bug fixes + +Analysis:: +* Adjust exception thrown when unable to load hunspell dict {es-pull}123743[#123743] +* Non existing synonyms sets do not fail shard recovery for indices {es-pull}125659[#125659] (issue: {es-issue}125603[#125603]) + +CAT APIs:: +* Fix cat_component_templates documentation {es-pull}120487[#120487] + +Data streams:: +* Avoid updating settings version in `MetadataMigrateToDataStreamService` when settings have not changed {es-pull}118704[#118704] +* Ensure removal of index blocks does not leave key with null value {es-pull}122246[#122246] +* Match dot prefix of migrated DS backing index with the source index {es-pull}120042[#120042] +* Refresh source index before reindexing data stream index {es-pull}120752[#120752] (issue: {es-issue}120314[#120314]) +* Updating `TransportRolloverAction.checkBlock` so that non-write-index blocks do not prevent data stream rollover {es-pull}122905[#122905] +* `ReindexDataStreamIndex` bug in assertion caused by reference equality {es-pull}121325[#121325] + +Downsampling:: +* Copy metrics and `default_metric` properties when downsampling `aggregate_metric_double` {es-pull}121727[#121727] (issues: {es-issue}119696[#119696], {es-issue}96076[#96076]) +* Improve downsample performance by avoiding to read unnecessary dimension values when downsampling. {es-pull}124451[#124451] + +EQL:: +* Fix EQL double invoking listener {es-pull}124918[#124918] + +ES|QL:: +* Avoid over collecting in Limit or Lucene Operator {es-pull}123296[#123296] +* Correct line and column numbers of missing named parameters {es-pull}120852[#120852] +* Drop null columns in text formats {es-pull}117643[#117643] (issue: {es-issue}116848[#116848]) +* {esql} - date nanos range bug? {es-pull}125345[#125345] (issue: {es-issue}125439[#125439]) +* {esql} - Fix lucene push down behavior when a range contains nanos and millis {es-pull}125595[#125595] +* Fix ROUND() with unsigned longs throwing in some edge cases {es-pull}119536[#119536] +* Fix TopN row size estimate {es-pull}119476[#119476] (issue: {es-issue}106956[#106956]) +* Fix `AbstractShapeGeometryFieldMapperTests` {es-pull}119265[#119265] (issue: {es-issue}119201[#119201]) +* Fix `ReplaceMissingFieldsWithNull` {es-pull}125764[#125764] (issues: {es-issue}126036[#126036], {es-issue}121754[#121754], {es-issue}126030[#126030]) +* Fix a bug in TOP {es-pull}121552[#121552] +* Fix async stop sometimes not properly collecting result {es-pull}121843[#121843] (issue: {es-issue}121249[#121249]) +* Fix attribute set equals {es-pull}118823[#118823] +* Fix double lookup failure on {esql} {es-pull}115616[#115616] (issue: {es-issue}111398[#111398]) +* Fix queries with document level security on lookup indexes {es-pull}120617[#120617] (issue: {es-issue}120509[#120509]) +* Fix writing for LOOKUP status {es-pull}119296[#119296] (issue: {es-issue}119086[#119086]) +* Implicit numeric casting for CASE/GREATEST/LEAST {es-pull}122601[#122601] (issue: {es-issue}121890[#121890]) +* Lazy collection copying during node transform {es-pull}124424[#124424] +* Limit memory usage of `fold` {es-pull}118602[#118602] +* Limit size of query {es-pull}117898[#117898] +* Make `numberOfChannels` consistent with layout map by removing duplicated `ChannelSet` {es-pull}125636[#125636] +* Reduce iteration complexity for plan traversal {es-pull}123427[#123427] +* Remove redundant sorts from execution plan {es-pull}121156[#121156] +* Use a must boolean statement when pushing down to Lucene when scoring is also needed {es-pull}124001[#124001] (issue: {es-issue}123967[#123967]) + +Health:: +* Do not recommend increasing `max_shards_per_node` {es-pull}120458[#120458] + +ILM+SLM:: +* Add missing timeouts to rest-api-spec SLM APIs {es-pull}119447[#119447] + +Indices APIs:: +* Fix NPE in rolling over unknown target and return 404 {es-pull}125352[#125352] +* Include hidden indices in `DeprecationInfoAction` {es-pull}118035[#118035] (issue: {es-issue}118020[#118020]) +* Preventing `ConcurrentModificationException` when updating settings for more than one index {es-pull}126077[#126077] +* Updates the deprecation info API to not warn about system indices and data streams {es-pull}122951[#122951] +* Avoid hoarding cluster state references during rollover {es-pull}124266[#124266] + +Inference:: +* [Inference API] Put back legacy EIS URL setting {es-pull}121207[#121207] + +Infra/Core:: +* Epoch Millis Rounding Down and Not Up 2 {es-pull}118353[#118353] +* Fix system data streams to be restorable from a snapshot {es-pull}124651[#124651] (issue: {es-issue}89261[#89261]) +* Have create index return a bad request on poor formatting {es-pull}123761[#123761] +* Include data streams when converting an existing resource to a system resource {es-pull}121392[#121392] +* System Index Migration Failure Results in a Non-Recoverable State {es-pull}122326[#122326] +* System data streams are not being upgraded in the feature migration API {es-pull}123926[#123926] +* Wrap jackson exception on malformed json string {es-pull}114445[#114445] (issue: {es-issue}114142[#114142]) + +Infra/Logging:: +* Move `SlowLogFieldProvider` instantiation to node construction {es-pull}117949[#117949] + +Infra/Plugins:: +* Remove unnecessary entitlement {es-pull}120959[#120959] +* Restrict agent entitlements to the system classloader unnamed module {es-pull}120546[#120546] + +Ingest Node:: +* Fix geoip databases index access after system feature migration {es-pull}121196[#121196] +* Fix geoip databases index access after system feature migration (again) {es-pull}122938[#122938] +* Fix geoip databases index access after system feature migration (take 3) {es-pull}124604[#124604] + +Machine Learning:: +* Add `ElasticInferenceServiceCompletionServiceSettings` {es-pull}123155[#123155] +* Add enterprise license check to inference action for semantic text fields {es-pull}122293[#122293] +* Avoid potentially throwing calls to Task#getDescription in model download {es-pull}124527[#124527] +* Change format for Unified Chat {es-pull}121396[#121396] +* Fix `AlibabaCloudSearchCompletionAction` not accepting `ChatCompletionInputs` {es-pull}125023[#125023] +* Fix get all inference endponts not returning multiple endpoints sharing model deployment {es-pull}121821[#121821] +* Fix serialising the inference update request {es-pull}122278[#122278] +* Fixing bedrock event executor terminated cache issue {es-pull}118177[#118177] (issue: {es-issue}117916[#117916]) +* Fixing bug setting index when parsing Google Vertex AI results {es-pull}117287[#117287] +* Retry on streaming errors {es-pull}123076[#123076] +* Set Connect Timeout to 5s {es-pull}123272[#123272] +* Set default similarity for Cohere model to cosine {es-pull}125370[#125370] (issue: {es-issue}122878[#122878]) +* Updating Inference Update API documentation to have the correct PUT method {es-pull}121048[#121048] +* [Inference API] Fix output stream ordering in `InferenceActionProxy` {es-pull}124225[#124225] + +Mapping:: +* Avoid serializing empty `_source` fields in mappings {es-pull}122606[#122606] +* Fix realtime get of nested fields with synthetic source {es-pull}119575[#119575] (issue: {es-issue}119553[#119553]) +* Merge field mappers when updating mappings with [subobjects:false] {es-pull}120370[#120370] (issue: {es-issue}120216[#120216]) +* Merge template mappings properly during validation {es-pull}124784[#124784] (issue: {es-issue}123372[#123372]) +* Tweak `copy_to` handling in synthetic `_source` to account for nested objects {es-pull}120974[#120974] (issue: {es-issue}120831[#120831]) + +Ranking:: +* Fix LTR query feature with phrases (and two-phase) queries {es-pull}125103[#125103] + +Search:: +* Catch and handle disconnect exceptions in search {es-pull}115836[#115836] +* Fix leak in `DfsQueryPhase` and introduce search disconnect stress test {es-pull}116060[#116060] (issue: {es-issue}115056[#115056]) +* Handle long overflow in dates {es-pull}124048[#124048] (issue: {es-issue}112483[#112483]) +* Handle search timeout in `SuggestPhase` {es-pull}122357[#122357] (issue: {es-issue}122186[#122186]) +* In this pr, a 400 error is returned when _source / _seq_no / _feature / _nested_path / _field_names is requested, rather a 5xx {es-pull}117229[#117229] +* Load `FieldInfos` from store if not yet initialised through a refresh on `IndexShard` {es-pull}125650[#125650] (issue: {es-issue}125483[#125483]) +* Re-enable parallel collection for field sorted top hits {es-pull}125916[#125916] +* Skip fetching _inference_fields field in legacy semantic_text format {es-pull}121720[#121720] +* Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version. {es-pull}125389[#125389] +* Test/107515 `RestoreTemplateWithMatchOnlyTextMapperIT` {es-pull}120898[#120898] +* Fix/SearchStatesIt_failures {es-pull}117729[#117729] +* `CrossClusterIT` `testCancel` failure {es-pull}117750[#117750] (issue: {es-issue}108061[#108061]) + +Snapshot/Restore:: +* Fork post-snapshot-delete cleanup off master thread {es-pull}122731[#122731] +* This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features. {es-pull}124931[#124931] +* Use the system index descriptor in the snapshot blob cache cleanup task {es-pull}120937[#120937] (issue: {es-issue}120518[#120518]) + +Suggesters:: +* Return an empty suggestion when suggest phase times out {es-pull}122575[#122575] (issue: {es-issue}122548[#122548]) + +Transform:: +* If the Transform is configured to write to an alias as its destination index, when the delete_dest_index parameter is set to true, then the Delete API will now delete the write index backing the alias {es-pull}122074[#122074] (issue: {es-issue}121913[#121913]) + +Vector Search:: +* Apply default k for knn query eagerly {es-pull}118774[#118774] +* Fix `bbq_hnsw` merge file cleanup on random IO exceptions {es-pull}119691[#119691] (issue: {es-issue}119392[#119392]) +* Knn vector rescoring to sort score docs {es-pull}122653[#122653] (issue: {es-issue}119711[#119711]) +* Return appropriate error on null dims update instead of npe {es-pull}125716[#125716] + +Watcher:: +* Watcher history index has too many indexed fields - {es-pull}117701[#117701] (issue: {es-issue}71479[#71479]) + +[[deprecation-8.18.0]] +[float] +=== Deprecations + +Indices APIs:: +* Increase the frozen indices deprecation level to `CRITICAL` {es-pull}119879[#119879] + +Infra/Core:: +* Add deprecation warning to `TransportHandshaker` {es-pull}123188[#123188] + +Infra/REST API:: +* Enhancement/v7 critical deprecation logging {es-pull}118298[#118298] + +[[enhancement-8.18.0]] +[float] +=== Enhancements + +Authentication:: +* Allow `SSHA-256` for API key credential hash {es-pull}120997[#120997] + +Authorization:: +* Allow kibana_system user to manage .reindexed-v8-internal.alerts indices {es-pull}118959[#118959] +* Do not fetch reserved roles from native store when Get Role API is called {es-pull}121971[#121971] +* Make reserved built-in roles queryable {es-pull}117581[#117581] +* [Security Solution] allows `kibana_system` user to manage .reindexed-v8-* Security Solution indices {es-pull}119054[#119054] + +CCS:: +* Resolve/cluster allows querying for cluster info only (no index expression required) {es-pull}119898[#119898] + +Data streams:: +* Add action to create index from a source index {es-pull}118890[#118890] +* Add index and reindex request settings to speed up reindex {es-pull}119780[#119780] +* Add rest endpoint for `create_from_source_index` {es-pull}119250[#119250] +* Add sanity check to `ReindexDatastreamIndexAction` {es-pull}120231[#120231] +* Adding a migration reindex cancel API {es-pull}118291[#118291] +* Adding get migration reindex status {es-pull}118267[#118267] +* Consistent mapping for OTel log and event bodies {es-pull}120547[#120547] +* Filter deprecated settings when making dest index {es-pull}120163[#120163] +* Ignore closed indices for reindex {es-pull}120244[#120244] +* Improve how reindex data stream index action handles api blocks {es-pull}120084[#120084] +* Initial work on `ReindexDatastreamIndexAction` {es-pull}116996[#116996] +* Make `requests_per_second` configurable to throttle reindexing {es-pull}120207[#120207] +* Optimized index sorting for OTel logs {es-pull}119504[#119504] +* Reindex data stream indices on different nodes {es-pull}125171[#125171] +* Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning {es-pull}120629[#120629] +* Retry ILM async action after reindexing data stream {es-pull}124149[#124149] +* Update data stream deprecations warnings to new format and filter sea… {es-pull}119097[#119097] + +Distributed:: +* Metrics for incremental bulk splits {es-pull}116765[#116765] + +Downsampling:: +* Improve downsample performance by buffering docids and do bulk processing {es-pull}124477[#124477] +* Improve rolling up metrics {es-pull}124739[#124739] + +EQL:: +* Add support for partial shard results {es-pull}116388[#116388] +* Optional named arguments for function in map {es-pull}118619[#118619] + +ES|QL:: +* Add ES|QL cross-cluster query telemetry collection {es-pull}119474[#119474] +* Add a `LicenseAware` interface for licensed Nodes {es-pull}118931[#118931] (issue: {es-issue}117405[#117405]) +* Add a `PostAnalysisAware,` distribute verification {es-pull}119798[#119798] +* Add a standard deviation aggregating function: STD_DEV {es-pull}116531[#116531] +* Add cluster level reduction {es-pull}117731[#117731] +* Add nulls support to Categorize {es-pull}117655[#117655] +* Async search responses have CCS metadata while searches are running {es-pull}117265[#117265] +* Backport Term query for ES|QL to 8.x {es-pull}118135[#118135] +* Backport scoring support in ES|QL to 8.x branch {es-pull}117747[#117747] +* Check for early termination in Driver {es-pull}118188[#118188] +* Do not serialize `EsIndex` in plan {es-pull}119580[#119580] +* {esql} - Remove restrictions for disjunctions in full text functions {es-pull}118544[#118544] +* {esql} - enabling scoring with METADATA `_score` {es-pull}113120[#113120] +* {esql} Add {esql} hash function {es-pull}117989[#117989] +* {esql} Support IN operator for Date nanos {es-pull}119772[#119772] (issue: {es-issue}118578[#118578]) +* {esql}: CATEGORIZE as a `BlockHash` {es-pull}114317[#114317] +* {esql}: Enterprise license enforcement for CCS {es-pull}118102[#118102] +* ES|QL: Partial result on demand for async queries {es-pull}118122[#118122] +* Enable KQL function as a tech preview {es-pull}119730[#119730] +* Enable LOOKUP JOIN in non-snapshot builds {es-pull}121193[#121193] (issue: {es-issue}121185[#121185]) +* Enable node-level reduction by default {es-pull}119621[#119621] +* Enable physical plan verification {es-pull}118114[#118114] +* {esql} - Support date nanos in date extract function {es-pull}120727[#120727] (issue: {es-issue}110000[#110000]) +* {esql} - support date nanos in date format function {es-pull}120143[#120143] (issue: {es-issue}109994[#109994]) +* {esql} Support date nanos on date diff function {es-pull}120645[#120645] (issue: {es-issue}109999[#109999]) +* {esql} bucket function for date nanos {es-pull}118474[#118474] (issue: {es-issue}118031[#118031]) +* {esql} compare nanos and millis {es-pull}118027[#118027] (issue: {es-issue}116281[#116281]) +* {esql} implicit casting for date nanos {es-pull}118697[#118697] (issue: {es-issue}118476[#118476]) +* Extend `TranslationAware` to all pushable expressions {es-pull}120192[#120192] +* Hash functions {es-pull}118938[#118938] +* Implement a `MetricsAware` interface {es-pull}121074[#121074] +* LOOKUP JOIN using field-caps for field mapping {es-pull}117246[#117246] +* Lookup join on multiple join fields not yet supported {es-pull}118858[#118858] +* Move scoring in ES|QL out of snapshot {es-pull}120354[#120354] +* Optimize ST_EXTENT_AGG for `geo_shape` and `cartesian_shape` {es-pull}119889[#119889] +* Push down `StartsWith` and `EndsWith` functions to Lucene {es-pull}123381[#123381] (issue: {es-issue}123067[#123067]) +* Push down filter passed lookup join {es-pull}118410[#118410] +* Resume Driver on cancelled or early finished {es-pull}120020[#120020] +* Reuse child `outputSet` inside the plan where possible {es-pull}124611[#124611] +* Rewrite TO_UPPER/TO_LOWER comparisons {es-pull}118870[#118870] (issue: {es-issue}118304[#118304]) +* ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape {es-pull}118802[#118802] +* Smarter field caps with subscribable listener {es-pull}116755[#116755] +* Support some stats on aggregate_metric_double {es-pull}120343[#120343] (issue: {es-issue}110649[#110649]) +* Take named parameters for identifier and pattern out of snapshot {es-pull}121850[#121850] +* Term query for ES|QL {es-pull}117359[#117359] +* Update grammar to rely on `indexPattern` instead of identifier in join target {es-pull}120494[#120494] +* `_score` should not be a reserved attribute in ES|QL {es-pull}118435[#118435] (issue: {es-issue}118460[#118460]) + +Engine:: +* Enhance add-block API to flush and add 'verified' metadata {es-pull}119743[#119743] + +Experiences:: +* Integrate IBM watsonx to Inference API for re-ranking task {es-pull}117176[#117176] + +Geo:: +* Optimize indexing points with index and doc values set to true {es-pull}120271[#120271] + +ILM+SLM:: +* Add a `replicate_for` option to the ILM `searchable_snapshot` action {es-pull}119003[#119003] + +Indices APIs:: +* Add `remove_index_block` arg to `_create_from` api {es-pull}120548[#120548] +* Remove index blocks by default in `create_from` {es-pull}120643[#120643] +* introduce new categories for deprecated resources in deprecation API {es-pull}120505[#120505] + +Inference:: +* [Inference API] Rename `model_id` prop to model in EIS sparse inference request body {es-pull}122398[#122398] +* Add version prefix to Inference Service API path {es-pull}117696[#117696] +* Update sparse text embeddings API route for Inference Service {es-pull}118369[#118369] +* [Elastic Inference Service] Add ElasticInferenceService Unified ChatCompletions Integration {es-pull}118871[#118871] + +Infra/CLI:: +* Ignore _JAVA_OPTIONS {es-pull}124843[#124843] +* Strengthen encryption for elasticsearch-keystore tool to AES 256 {es-pull}119749[#119749] + +Infra/Core:: +* Improve size limiting string message {es-pull}122427[#122427] +* Return unique deprecation for old indices with incompatible date formats {es-pull}124597[#124597] + +Infra/REST API:: +* A new query parameter `?include_source_on_error` was added for create / index, update and bulk REST APIs to control +if to include the document source in the error response in case of parsing errors. The default value is `true`. {es-pull}120725[#120725] + +Ingest Node:: +* Optimize `IngestCtxMap` construction {es-pull}120833[#120833] +* Optimize `IngestDocMetadata` `isAvailable` {es-pull}120753[#120753] +* Optimize `IngestDocument` `FieldPath` allocation {es-pull}120573[#120573] +* Optimize some per-document hot paths in the geoip processor {es-pull}120824[#120824] +* Returning ignored fields in the simulate ingest API {es-pull}117214[#117214] + +Logs:: +* Add LogsDB option to route on sort fields {es-pull}116687[#116687] +* Add a new index setting to skip recovery source when synthetic source is enabled {es-pull}114618[#114618] +* Configure index sorting through index settings for logsdb {es-pull}118968[#118968] (issue: {es-issue}118686[#118686]) +* Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on. {es-pull}120055[#120055] + +Machine Learning:: +* Add Inference Unified API for chat completions for OpenAI {es-pull}117589[#117589] +* Add Jina AI API to do inference for Embedding and Rerank models {es-pull}118652[#118652] +* Add enterprise license check for Inference API actions {es-pull}119893[#119893] +* Adding chunking settings to `IbmWatsonxService` {es-pull}114914[#114914] +* Adding default endpoint for Elastic Rerank {es-pull}117939[#117939] +* Adding endpoint creation validation for all task types to remaining services {es-pull}115020[#115020] +* Check for presence of error object when validating streaming responses from integrations in the inference API {es-pull}118375[#118375] +* Ignore failures from renormalizing buckets in read-only index {es-pull}118674[#118674] +* Inference duration and error metrics {es-pull}115876[#115876] +* Migrate stream to core error parsing {es-pull}120722[#120722] +* Remove all mentions of eis and gateway and deprecate flags that do {es-pull}116692[#116692] +* Remove deprecated sort from reindex operation within dataframe analytics procedure {es-pull}117606[#117606] +* Retry on `ClusterBlockException` on transform destination index {es-pull}118194[#118194] + +Mapping:: +* Add Optional Source Filtering to Source Loaders {es-pull}113827[#113827] + +Network:: +* Allow http unsafe buffers by default {es-pull}116115[#116115] +* Http stream activity tracker and exceptions handling {es-pull}119564[#119564] +* Remove HTTP content copies {es-pull}117303[#117303] +* `ConnectTransportException` returns retryable BAD_GATEWAY {es-pull}118681[#118681] (issue: {es-issue}118320[#118320]) + +Ranking:: +* Set default reranker for text similarity reranker to Elastic reranker {es-pull}120551[#120551] + +Search:: +* Add match support for `semantic_text` fields {es-pull}117839[#117839] +* Add support for `sparse_vector` queries against `semantic_text` fields {es-pull}118617[#118617] +* Add support for knn vector queries on `semantic_text` fields {es-pull}119011[#119011] +* Adding linear retriever to support weighted sums of sub-retrievers {es-pull}120222[#120222] +* Feat: add a user-configurable timeout parameter to the `_resolve/cluster` API {es-pull}120542[#120542] +* Make semantic text part of the text family {es-pull}119792[#119792] +* Only aggregations require at least one shard request {es-pull}115314[#115314] +* Prevent data nodes from sending stack traces to coordinator when `error_trace=false` {es-pull}118266[#118266] +* Propagate status codes from shard failures appropriately {es-pull}118016[#118016] (issue: {es-issue}118482[#118482]) + +Snapshot/Restore:: +* Add IMDSv2 support to `repository-s3` {es-pull}117748[#117748] (issue: {es-issue}105135[#105135]) + +Store:: +* Abort pending deletion on `IndicesService` close {es-pull}123569[#123569] + +TSDB:: +* Increase field limit for OTel metrics to 10 000 {es-pull}120591[#120591] + +Transform:: +* Add support for `extended_stats` {es-pull}120340[#120340] +* Auto-migrate `max_page_search_size` {es-pull}119348[#119348] +* Create upgrade mode {es-pull}117858[#117858] +* Wait while index is blocked {es-pull}119542[#119542] +* [Deprecation] Add `transform_ids` to outdated index {es-pull}120821[#120821] + +Vector Search:: +* Even better(er) binary quantization {es-pull}117994[#117994] +* Speed up bit compared with floats or bytes script operations {es-pull}117199[#117199] + +[[feature-8.18.0]] +[float] +=== New features + +CRUD:: +* Metrics for indexing failures due to version conflicts {es-pull}119067[#119067] + +ES|QL:: +* {esql} - Add Match function options {es-pull}120360[#120360] +* {esql} - Allow full text functions disjunctions for non-full text functions {es-pull}120291[#120291] +* {esql}: Enable async get to support formatting {es-pull}111104[#111104] (issue: {es-issue}110926[#110926]) +* Expand type compatibility for match function and operator {es-pull}117555[#117555] +* ST_EXTENT aggregation {es-pull}117451[#117451] (issue: {es-issue}104659[#104659]) +* Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions {es-pull}116964[#116964] (issue: {es-issue}104875[#104875]) + +Highlighting:: +* Add Highlighter for Semantic Text Fields {es-pull}118064[#118064] + +Infra/Core:: +* Infrastructure for assuming cluster features in the next major version {es-pull}118143[#118143] + +Machine Learning:: +* ES|QL categorize with multiple groupings {es-pull}118173[#118173] +* Support mTLS for the Elastic Inference Service integration inside the inference API {es-pull}119679[#119679] +* [Inference API] Add node-local rate limiting for the inference API {es-pull}120400[#120400] + +Mapping:: +* Add option to store `sparse_vector` outside `_source` {es-pull}117917[#117917] +* Release semantic_text as a GA feature {es-pull}124670[#124670] + +Ranking:: +* Add a generic `rescorer` retriever based on the search request's rescore functionality {es-pull}118585[#118585] (issue: {es-issue}118327[#118327]) + +Relevance:: +* Add Multi-Field Support for Semantic Text Fields {es-pull}120128[#120128] + +Vector Search:: +* Add new experimental `rank_vectors` mapping for late-interaction second order ranking {es-pull}118804[#118804] +* KNN vector rescoring for quantized vectors {es-pull}116663[#116663] +* Mark bbq indices as GA and add rolling upgrade integration tests {es-pull}121105[#121105] +* Add new experimental `rank_vectors` mapping for late-interaction second order ranking {es-pull}119601[#119601] + +[[upgrade-8.18.0]] +[float] +=== Upgrades + +Infra/Core:: +* Bump major version for feature migration system indices {es-pull}117243[#117243] +* Permanently switch from Java SecurityManager to Entitlements. The Java SecurityManager has been deprecated since Java 17, and it is now completely disabled in Java 24. In order to retain an similar level of protection, Elasticsearch implemented its own protection mechanism, Entitlements. Starting with this version, Entitlements will permanently replace the Java SecurityManager. {es-pull}125073[#125073] +* Update ASM 9.7 -> 9.7.1 to support JDK 24 {es-pull}118094[#118094] + +Machine Learning:: +* Automatically rollover legacy .ml-anomalies indices {es-pull}120885[#120885] +* Automatically rollover legacy ml indices {es-pull}120405[#120405] +* Change the auditor to write via an alias {es-pull}120064[#120064] +* Check if the anomaly results index has been rolled over {es-pull}125404[#125404] +* Update minimum supported snapshot version for Machine Learning jobs to 8.3.0 {es-pull}118166[#118166] + +Packaging:: +* Update bundled JDK to Java 24 {es-pull}125159[#125159] + +Search:: +* Upgrade to Lucene 9.12.1 {es-pull}118300[#118300] + +Watcher:: +* Script for migrating `.watches` and `.triggered_watches` indices {es-pull}120371[#120371] + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index a3fefb88019af..67310d0d069b8 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -36,18 +36,30 @@ endif::[] [discrete] [[add_new_experimental_rank_vectors_mapping_for_late_interaction_second_order_ranking]] -=== Add new experimental `rank_vectors` mapping for late-interaction second order ranking +=== `rank_vectors` field type is now available for late-interaction ranking + +<> is a new field type released as an experimental feature in {es} {minor-version}. It is designed to be used with dense vectors and allows for late-interaction second order ranking. + Late-interaction models are powerful rerankers. While their size and overall cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali. -{es-pull}118804[#118804] +[discrete] +[[enable_lookup_join_in_non_snapshot_builds]] +=== {esql} LOOKUP JOIN is now available in technical preview + +<> is now available in technical preview. +LOOKUP JOIN combines data from your {esql} queries with matching records from a lookup index, enabling you to: + +* Enrich your search results with reference data +* Speed up root-cause analysis and security investigations +* Join data across indices without complex queries +* Reduce operational overhead when correlating events [discrete] -[[8_x_add_new_experimental_rank_vectors_mapping_for_late_interaction_second_order_ranking]] -=== [8.x] Add new experimental `rank_vectors` mapping for late-interaction second order ranking -Backports the following commits to 8.x: - Add new experimental -rank_vectors mapping for late-interaction second order ranking (#118804) +[[release_semantic_text_as_ga_feature]] +=== The `semantic_text` field type is now GA + +<> is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time. -{es-pull}119601[#119601] // end::notable-highlights[] diff --git a/docs/reference/reranking/semantic-reranking.asciidoc b/docs/reference/reranking/semantic-reranking.asciidoc index e1e2abd224a8e..2557c7604628b 100644 --- a/docs/reference/reranking/semantic-reranking.asciidoc +++ b/docs/reference/reranking/semantic-reranking.asciidoc @@ -1,8 +1,6 @@ [[semantic-reranking]] == Semantic re-ranking -preview::[] - [TIP] ==== This overview focuses more on the high-level concepts and use cases for semantic re-ranking. For full implementation details on how to set up and use semantic re-ranking in {es}, see the <> in the Search API docs. @@ -87,11 +85,11 @@ To use semantic re-ranking in {es}, you need to: . *Select and configure a re-ranking model*. You have the following options: -.. Use the <> cross-encoder model via the inference API's {es} service. +.. Use the <> cross-encoder model via a preconfigured `.rerank-v1-elasticsearch` or by creating a custom deployment using the inference API's {es} service. .. Use the <> to create a `rerank` endpoint. .. Use the <> to create a `rerank` endpoint. .. Upload a model to {es} from Hugging Face with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland]. You'll need to use the `text_similarity` NLP task type when loading the model using Eland. Then set up an <> with the `rerank` endpoint type. -+ ++ Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third party text similarity models supported by {es} for semantic re-ranking. . *Create a `rerank` endpoint using the <>*. @@ -137,4 +135,4 @@ POST _search * Read the <> for syntax and implementation details * Learn more about the <> abstraction * Learn more about the Elastic <> -* Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es} \ No newline at end of file +* Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es} diff --git a/docs/reference/rest-api/watcher/update-settings.asciidoc b/docs/reference/rest-api/watcher/update-settings.asciidoc index 9ad38064e34ab..c0ab23230d064 100644 --- a/docs/reference/rest-api/watcher/update-settings.asciidoc +++ b/docs/reference/rest-api/watcher/update-settings.asciidoc @@ -11,10 +11,17 @@ For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. -- -This API allows a user to modify the settings for the Watcher internal index (`.watches`). Only a subset of settings are allowed to by modified. This includes: +This API allows a user to modify the settings for the Watcher internal index (`.watches`). Only a subset of settings +are allowed to be modified. This includes: - `index.auto_expand_replicas` - `index.number_of_replicas` +- Any setting with the prefix `index.routing.allocation.exclude.` +- Any setting with the prefix `index.routing.allocation.include.` +- Any setting with the prefix `index.routing.allocation.require.` + +Modification of `index.routing.allocation.include._tier_preference` is an exception and is not allowed as the Watcher +shards must always be in the `data_content` tier. An example of modifying the Watcher settings: diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index 2c8ef618dfa00..5d39154bfc1a3 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -536,13 +536,14 @@ Refer to <> for a high level overview of semantic re-ranking ===== Prerequisites -To use `text_similarity_reranker` you must first set up an inference endpoint for the `rerank` task using the <>. -The endpoint should be set up with a machine learning model that can compute text similarity. -Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third-party text similarity models supported by {es}. +To use `text_similarity_reranker`, you can rely on the preconfigured `.rerank-v1-elasticsearch` inference endpoint, which is based on <> and serves as the default if no `inference_id` is provided. +This model is optimized for reranking based on text similarity. If you'd like to use a different model, you can set up a custom inference endpoint for the `rerank` task using the <>. +The endpoint should be configured with a machine learning model capable of computing text similarity. +Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third-party text similarity models supported by {{es}}. You have the following options: -* Use the the built-in <> cross-encoder model via the inference API's {es} service. +* Use the built-in <> cross-encoder model via the inference API’s {{es}} service. For an example of creating an endpoint using the Elastic Rerank model, refer to <>. * Use the <> with the `rerank` task type. * Use the <> with the `rerank` task type. * Upload a model to {es} with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland] using the `text_similarity` NLP task type. @@ -582,9 +583,9 @@ The document field to be used for text similarity comparisons. This field should contain the text that will be evaluated against the `inferenceText`. `inference_id`:: -(Required, `string`) +(Optional, `string`) + -Unique identifier of the inference endpoint created using the {infer} API. +Unique identifier of the inference endpoint created using the {infer} API. If you don’t specify an inference endpoint, the `inference_id` field defaults to `.rerank-v1-elasticsearch`, a preconfigured endpoint for the elasticsearch `.rerank-v1` model. `inference_text`:: (Required, `string`) diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc index a68f20fb1c656..6ba587c12e97b 100644 --- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc +++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc @@ -1,26 +1,27 @@ |==== -| 21+^h| Remote cluster version +| 23+^h| Remote cluster version h| Local cluster version - | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 | 8.15 | 8.16 | 8.17 -| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.15 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.16 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.17 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} + | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 | 8.15 | 8.16 | 8.17 | 8.18 | +| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | +| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | +| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.15 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.16 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.17 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | +| 8.18 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | |==== diff --git a/docs/reference/search/search-your-data/cohere-es.asciidoc b/docs/reference/search/search-your-data/cohere-es.asciidoc index 3029cfd9f098c..cea17a4ed9a87 100644 --- a/docs/reference/search/search-your-data/cohere-es.asciidoc +++ b/docs/reference/search/search-your-data/cohere-es.asciidoc @@ -297,7 +297,7 @@ Rerank the results using the new {infer} endpoint. [source,py] -------------------------------------------------- # Pass the query and the search results to the service -response = client.inference.inference( +response = client.inference.rerank( inference_id="cohere_rerank", body={ "query": query, diff --git a/docs/reference/search/search-your-data/highlighting.asciidoc b/docs/reference/search/search-your-data/highlighting.asciidoc index 16546840a7828..cae569820980f 100644 --- a/docs/reference/search/search-your-data/highlighting.asciidoc +++ b/docs/reference/search/search-your-data/highlighting.asciidoc @@ -276,9 +276,11 @@ max_analyzed_offset:: By default, the maximum number of characters analyzed for a highlight request is bounded by the value defined in the <> setting, and when the number of characters exceeds this limit an error is returned. If -this setting is set to a non-negative value, the highlighting stops at this defined +this setting is set to a positive value, the highlighting stops at this defined maximum limit, and the rest of the text is not processed, thus not highlighted and -no error is returned. The <> query setting +no error is returned. If it is specifically set to -1 then the value of +<> is used instead. +For values < -1 or 0, an error is returned. The <> query setting does *not* override the <> which prevails when it's set to lower value than the query setting. diff --git a/docs/reference/search/search-your-data/paginate-search-results.asciidoc b/docs/reference/search/search-your-data/paginate-search-results.asciidoc index f69fd60be0484..ffb79bf9cb595 100644 --- a/docs/reference/search/search-your-data/paginate-search-results.asciidoc +++ b/docs/reference/search/search-your-data/paginate-search-results.asciidoc @@ -1,6 +1,19 @@ [[paginate-search-results]] === Paginate search results +{es} uses pagination to segment large result sets into manageable pages for efficient retrieval and processing. + +{es} supports three pagination techniques: + + +* <>: Ideal for creating a list of pages that users can navigate. +* <>: Supports infinite scroll or allows loading additional results with a "next" button. +* <>: Historically used to retrieve all matching documents. The <> method with the <> is now recommended for better efficiency and reliability. + +[discrete] +[[from-and-size-pagination]] +=== From and size pagination + By default, searches return the top 10 matching hits. To page through a larger set of results, you can use the <>'s `from` and `size` parameters. The `from` parameter defines the number of hits to skip, defaulting @@ -25,7 +38,7 @@ Avoid using `from` and `size` to page too deeply or request too many results at once. Search requests usually span multiple shards. Each shard must load its requested hits and the hits for any previous pages into memory. For deep pages or large sets of results, these operations can significantly increase memory and -CPU usage, resulting in degraded performance or node failures. +CPU usage. If not properly managed, these operations can result in degraded performance or node failures. By default, you cannot use `from` and `size` to page through more than 10,000 hits. This limit is a safeguard set by the @@ -33,6 +46,8 @@ hits. This limit is a safeguard set by the to page through more than 10,000 hits, use the <> parameter instead. +Pagination is stateless, so search result order may change when navigating between pages. To maintain consistent order, use the <> for stateful pagination. + WARNING: {es} uses Lucene's internal doc IDs as tie-breakers. These internal doc IDs can be completely different across replicas of the same data. When paging search hits, you might occasionally see that documents with the same sort values @@ -43,7 +58,7 @@ are not ordered consistently. === Search after You can use the `search_after` parameter to retrieve the next page of hits -using a set of <> from the previous page. +using a set of <> from the previous page. This approach is ideal for scenarios where users click a "next" or "load more" button, rather than selecting a specific page. Using `search_after` requires multiple search requests with the same `query` and `sort` values. The first step is to run an initial request. The following @@ -629,4 +644,4 @@ GET /my-index-000001/_search?scroll=1m -------------------------------------------------- // TEST[setup:my_index_big] -For append only time-based indices, the `timestamp` field can be used safely. +For append only time-based indices, the `timestamp` field can be used safely. \ No newline at end of file diff --git a/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc b/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc index 2958999ede91d..d04643635f33d 100644 --- a/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc +++ b/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc @@ -68,7 +68,7 @@ try the https://www.elastic.co/demo-gallery/ai-playground[interactive lab] for h Learn more about building RAG systems using {es} in these blog posts: -* https://www.elastic.co/blog/beyond-rag-basics-semantic-search-with-elasticsearch[Beyond RAG Basics: Advanced strategies for AI applications] +* https://www.elastic.co/blog/beyond-rag-basics[Beyond RAG Basics: Advanced strategies for AI applications] * https://www.elastic.co/search-labs/blog/building-a-rag-system-with-gemma-hugging-face-elasticsearch[Building a RAG system with Gemma, Hugging Face, and Elasticsearch] * https://www.elastic.co/search-labs/blog/rag-agent-tool-elasticsearch-langchain[Building an agentic RAG tool with Elasticsearch and Langchain] diff --git a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc index 5309b24fa37c9..c0e5d0a29ff9b 100644 --- a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc @@ -187,7 +187,7 @@ GET my-index/_search "query":{ "sparse_vector":{ "field": "content_embedding", - "inference_id": "my-elser-endpoint", + "inference_id": "my-elser-endpoint", "query": "How to avoid muscle soreness after running?" } } diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index 50a9da4af2fba..07bb90889716d 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -4,8 +4,6 @@ Semantic search with `semantic_text` ++++ -beta[] - This tutorial shows you how to use the semantic text feature to perform semantic search on your data. Semantic text simplifies the {infer} workflow by providing {infer} at ingestion time and sensible default values automatically. @@ -21,7 +19,7 @@ This tutorial uses the <> f [[semantic-text-requirements]] ==== Requirements -This tutorial uses the <> for demonstration, which is created automatically as needed. +This tutorial uses the <> for demonstration, which is created automatically as needed. To use the `semantic_text` field type with an {infer} service other than `elasticsearch` service, you must create an inference endpoint using the <>. @@ -92,7 +90,7 @@ The reindexed data will be processed by the {infer} endpoint associated with the ------------------------------------------------------------ POST _reindex?wait_for_completion=false { - "source": { + "source": { "index": "test-data", "size": 10 <1> }, @@ -130,7 +128,12 @@ POST _tasks//_cancel [[semantic-text-semantic-search]] ==== Semantic search -After the data set has been enriched with the embeddings, you can query the data using semantic search. +After the data set has been enriched with the embeddings, you can query the data using semantic search. You can use Query DSL or {esql} syntax. + +[discrete] +[[semantic-text-semantic-search-query-dsl]] +===== Query DSL syntax + Provide the `semantic_text` field name and the query text in a `semantic` query type. The {infer} endpoint used to generate the embeddings for the `semantic_text` field will be used to process the query text. @@ -139,7 +142,7 @@ The {infer} endpoint used to generate the embeddings for the `semantic_text` fie GET semantic-embeddings/_search { "query": { - "semantic": { + "semantic": { "field": "content", <1> "query": "How to avoid muscle soreness while running?" <2> } @@ -153,6 +156,35 @@ GET semantic-embeddings/_search As a result, you receive the top 10 documents that are closest in meaning to the query from the `semantic-embedding` index. +[discrete] +[[semantic-text-semantic-search-esql]] +===== {esql} syntax + +The ES|QL approach uses the <>, which automatically detects the `semantic_text` field and performs the search on it. The query uses `METADATA _score` to sort by `_score` in descending order. + +[source,console] +---- +POST /_query?format=txt +{ + "query": """ + FROM semantic-embeddings METADATA _score <1> + | WHERE content: "How to avoid muscle soreness while running?" <2> + | SORT _score DESC <3> + | LIMIT 1000 <4> + """ +} +---- +// TEST[skip:uses ML] +<1> The `METADATA _score` clause is used to return the score of each document +<2> The <> is used on the `content` field for standard keyword matching +<3> Sorts by descending score to display the most relevant results first +<4> Limits the results to 1000 documents + +[TIP] +==== +Refer to <> for more information on using the {esql} language for search use cases. +==== + [discrete] [[semantic-text-further-examples]] ==== Further examples and reading diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc index e0fb8415fee18..96700ca0afe84 100644 --- a/docs/reference/search/search-your-data/semantic-search.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search.asciidoc @@ -96,6 +96,7 @@ IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer ** https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[Semantic search with `semantic_text`] * Blogs: ** https://www.elastic.co/search-labs/blog/semantic-search-simplified-semantic-text[{es} new semantic_text mapping: Simplifying semantic search] +** https://www.elastic.co/search-labs/blog/semantic-text-ga[`semantic_text` format changes and new features in 8.18] ** {blog-ref}may-2023-launch-sparse-encoder-ai-model[Introducing Elastic Learned Sparse Encoder: Elastic's AI model for semantic search] ** {blog-ref}lexical-ai-powered-search-elastic-vector-database[How to get the best of lexical and AI-powered search with Elastic's vector database] ** Information retrieval blog series: diff --git a/docs/reference/search/search-your-data/semantic-text-hybrid-search b/docs/reference/search/search-your-data/semantic-text-hybrid-search index 4b49a7c3155db..23a04251c6989 100644 --- a/docs/reference/search/search-your-data/semantic-text-hybrid-search +++ b/docs/reference/search/search-your-data/semantic-text-hybrid-search @@ -24,7 +24,7 @@ PUT semantic-embeddings "mappings": { "properties": { "semantic_text": { <1> - "type": "semantic_text", + "type": "semantic_text" }, "content": { <2> "type": "text", @@ -112,7 +112,15 @@ POST _tasks//_cancel [[hybrid-search-perform-search]] ==== Perform hybrid search -After reindexing the data into the `semantic-embeddings` index, you can perform hybrid search by using <>. RRF is a technique that merges the rankings from both semantic and lexical queries, giving more weight to results that rank high in either search. This ensures that the final results are balanced and relevant. +After reindexing the data into the `semantic-embeddings` index, you can perform hybrid search. You can use retrievers syntax or {esql} syntax to perform the search. + +[discrete] +[[hybrid-search-retrievers-syntax]] +===== Retrievers syntax + +This approach uses the <> algorithm. RRF is a technique that merges the rankings from both semantic and lexical queries, giving more weight to results that rank high in either search. This ensures that the final results are balanced and relevant. + +To extract the most relevant fragments from the original text and query, you can use the <>: [source,console] ------------------------------------------------------------ @@ -215,3 +223,32 @@ After performing the hybrid search, the query will return the top 10 documents t } ------------------------------------------------------------ // NOTCONSOLE + +[discrete] +[[hybrid-search-esql-syntax]] +===== {esql} syntax + +The <> approach uses a combination of the match operator `:` and the match function `match()` to perform hybrid search. + +[source,console] +---- +POST /_query?format=txt +{ + "query": """ + FROM semantic-embeddings METADATA _score <1> + | WHERE content: "muscle soreness running?" OR match(semantic_text, "How to avoid muscle soreness while running?", { "boost": 0.75 }) <2> <3> + | SORT _score DESC <4> + | LIMIT 1000 + """ +} +---- +// TEST[skip:uses ML] +<1> The `METADATA _score` clause is used to return the score of each document +<2> The <> is used on the `content` field for standard keyword matching +<3> Semantic search using the `match()` function on the `semantic_text` field with a boost of `0.75` +<4> Sorts by descending score and limits to 1000 results + +[TIP] +==== +Refer to <> for more information on using the {esql} language for search use cases. +==== \ No newline at end of file diff --git a/docs/reference/security/authorization/built-in-roles.asciidoc b/docs/reference/security/authorization/built-in-roles.asciidoc index 6db08b307f193..18a45e3b03663 100644 --- a/docs/reference/security/authorization/built-in-roles.asciidoc +++ b/docs/reference/security/authorization/built-in-roles.asciidoc @@ -75,7 +75,7 @@ use of the {inference} APIs. Grants the `manage_inference` cluster privilege. [[built-in-roles-inference-user]] `inference_user`:: Provides the minimum privileges required to view {inference} configurations -and perform inference. Grants the `monintor_inference` cluster privilege. +and perform inference. Grants the `monitor_inference` cluster privilege. [[built-in-roles-ingest-user]] `ingest_admin` :: Grants access to manage *all* index templates and *all* ingest pipeline configurations. diff --git a/docs/reference/setup/advanced-configuration.asciidoc b/docs/reference/setup/advanced-configuration.asciidoc index f64adde3e4845..d65335d49760b 100644 --- a/docs/reference/setup/advanced-configuration.asciidoc +++ b/docs/reference/setup/advanced-configuration.asciidoc @@ -153,7 +153,6 @@ options. We do not recommend using `ES_JAVA_OPTS` in production. NOTE: If you are running {es} as a Windows service, you can change the heap size using the service manager. See <>. -[[heap-dump-path]] include::important-settings/heap-dump-path.asciidoc[leveloffset=-1] [[gc-logging]] diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index 26f9c79cb6693..b822ee9b3f903 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -41,6 +41,7 @@ include::important-settings/discovery-settings.asciidoc[] include::important-settings/heap-size.asciidoc[] +[[heap-dump-path]] include::important-settings/heap-dump-path.asciidoc[] include::important-settings/gc-logging.asciidoc[] diff --git a/docs/reference/sql/functions/like-rlike.asciidoc b/docs/reference/sql/functions/like-rlike.asciidoc index 23fe0f30fb568..ba98e466aed96 100644 --- a/docs/reference/sql/functions/like-rlike.asciidoc +++ b/docs/reference/sql/functions/like-rlike.asciidoc @@ -84,6 +84,7 @@ IMPORTANT: Even though `RLIKE` is a valid option when searching or filtering in When using `LIKE`/`RLIKE`, do consider using <> which are faster, much more powerful and offer the option of sorting by relevancy (results can be returned based on how well they matched). +//// For example: [cols="> * <> * <> +* <> * <> * <> * <> @@ -109,6 +110,8 @@ include::troubleshooting/data/increase-shard-limit.asciidoc[] include::troubleshooting/data/increase-cluster-shard-limit.asciidoc[] +include::troubleshooting/data/source-mode-setting.asciidoc[] + include::troubleshooting/corruption-issues.asciidoc[] include::troubleshooting/disk/fix-data-node-out-of-disk.asciidoc[] diff --git a/docs/reference/troubleshooting/data/source-mode-setting.asciidoc b/docs/reference/troubleshooting/data/source-mode-setting.asciidoc new file mode 100644 index 0000000000000..0ab6327120436 --- /dev/null +++ b/docs/reference/troubleshooting/data/source-mode-setting.asciidoc @@ -0,0 +1,49 @@ +[[troubleshoot-migrate-source-mode]] +== Configuring source mode in mappings is deprecated and replaced by an index setting + +Index <> mode was previously configured in mappings as follows: + +[source,js] +---- +"mappings": { + "source": { + "mode": "synthetic" + }, + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } +} +---- +// NOTCONSOLE + +Starting with version 8.18, this method for configuring the source mode is considered +deprecated and replaced by index setting `index.mapping.source.mode` that accepts the +same values: `stored`, `synthetic` and `disabled`. The index setting can be +used as follows: + +[source,js] +---- +"settings": { + "index.mapping.source.mode": "synthetic" +} +"mappings": { + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } +} +---- +// NOTCONSOLE + +Existing indexes using the deprecated method for configuring the source mode are not +affected, but creating new indexes raises warnings and will not be supported in a future +release. This can be an issue for data streams and other indexes that get regularly +generated using component templates. To avoid these problems, identify all affected +component templates, as shown in the <>, +and update them to use setting `index.mapping.source.mode` instead of +`mappings.source.mode`, according to the examples above. diff --git a/docs/reference/upgrade.asciidoc b/docs/reference/upgrade.asciidoc index d5057d9b87d85..8577ddbef579a 100644 --- a/docs/reference/upgrade.asciidoc +++ b/docs/reference/upgrade.asciidoc @@ -25,6 +25,21 @@ proceeding with the upgrade. For instructions, refer to {stack-ref}/upgrading-elastic-stack.html#prepare-to-upgrade[Prepare to upgrade from 7.x]. +[discrete] +[[upgrade-newer-releases]] +=== Out-of-order releases + +Elastic maintains several minor versions of {es} at once. This means releases +do not always happen in order of their version numbers. You can only upgrade to +{version} if the version you are currently running meets both of these +conditions: + +* Has an older version number than {version}. +* Has an earlier release date than {version}. + +If you are currently running a version with an older version number but a later +release date than {version}, wait for a newer release before upgrading. + [discrete] [[upgrade-index-compatibility]] === Index compatibility diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc index 25ac0f3a06a2a..b2fadb107f3ba 100644 --- a/docs/resiliency/index.asciidoc +++ b/docs/resiliency/index.asciidoc @@ -118,9 +118,10 @@ in the case of each type of failure. The plan is to have a test case that valida [discrete] === Run Jepsen (STATUS: ONGOING) -We have ported the known scenarios in the Jepsen blogs that check loss of acknowledged writes to our testing infrastructure. -The new tests are run continuously in our testing farm and are passing. We are also working on running Jepsen independently to verify -that no failures are found. +We have ported the known scenarios in the Jepsen blogs that check loss of +acknowledged writes to our testing infrastructure. The new tests are run +continuously in our testing farm and are passing. We will also monitor for new +failure scenarios and adapt our test suite as needed. == Completed diff --git a/gradle.properties b/gradle.properties index aa38a61ab0057..7c781d859cea6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,3 +19,6 @@ org.gradle.dependency.verification.console=verbose # allow user to specify toolchain via the RUNTIME_JAVA_HOME environment variable org.gradle.java.installations.fromEnv=RUNTIME_JAVA_HOME + +# if configuration cache enabled then enable parallel support too +org.gradle.configuration-cache.parallel=true diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 05fda8e0244de..9b5230f7d47a1 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -11,16 +11,15 @@ apache-compress = "org.apache.commons:commons-compress:1.26.1" apache-rat = "org.apache.rat:apache-rat:0.11" asm = { group = "org.ow2.asm", name="asm", version.ref="asm" } asm-tree = { group = "org.ow2.asm", name="asm-tree", version.ref="asm" } -bytebuddy = "net.bytebuddy:byte-buddy:1.14.12" +bytebuddy = "net.bytebuddy:byte-buddy:1.15.11" checkstyle = "com.puppycrawl.tools:checkstyle:10.3" -commons-codec = "commons-codec:commons-codec:1.11" commmons-io = "commons-io:commons-io:2.2" docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5" forbiddenApis = "de.thetaphi:forbiddenapis:3.8" -gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.18.1" +gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.19.2" hamcrest = "org.hamcrest:hamcrest:2.1" -httpcore = "org.apache.httpcomponents:httpcore:4.4.12" -httpclient = "org.apache.httpcomponents:httpclient:4.5.14" +httpcore5 = "org.apache.httpcomponents.core5:httpcore5:5.3.3" +httpclient5 = "org.apache.httpcomponents.client5:httpclient5:5.4.2" idea-ext = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4" javaparser = "com.github.javaparser:javaparser-core:3.18.0" json-schema-validator = "com.networknt:json-schema-validator:1.0.72" @@ -47,3 +46,6 @@ spock-platform = { group = "org.spockframework", name="spock-bom", version.ref=" spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0" wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2" xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2" + +[plugins] +ospackage = { id = "com.netflix.nebula.ospackage-base", version = "11.11.1" } diff --git a/gradle/gradle-daemon-jvm.properties b/gradle/gradle-daemon-jvm.properties new file mode 100644 index 0000000000000..188d373d6bc6f --- /dev/null +++ b/gradle/gradle-daemon-jvm.properties @@ -0,0 +1,8 @@ +#This file is generated by updateDaemonJvm +toolchainUrl.LINUX.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/linux/aarch64/jdk/hotspot/normal/eclipse?project\=jdk +toolchainUrl.LINUX.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/linux/x64/jdk/hotspot/normal/eclipse?project\=jdk +toolchainUrl.MAC_OS.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/mac/aarch64/jdk/hotspot/normal/eclipse?project\=jdk +toolchainUrl.MAC_OS.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/mac/x64/jdk/hotspot/normal/eclipse?project\=jdk +toolchainUrl.WINDOWS.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/windows/x64/jdk/hotspot/normal/eclipse?project\=jdk +toolchainVendor=ADOPTIUM +toolchainVersion=21 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 7270f12515b67..3651b7955fab8 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -69,9 +69,9 @@ - - - + + + @@ -809,14 +809,9 @@ - - - - - - - - + + + @@ -944,9 +939,9 @@ - - - + + + @@ -979,36 +974,24 @@ + + + + + - - - - - - - - - - - - - - - - - - - - + + + @@ -1416,114 +1399,74 @@ - - - - - - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + @@ -1719,9 +1662,9 @@ - - - + + + @@ -1774,9 +1717,9 @@ - - - + + + @@ -1784,24 +1727,14 @@ - - - - - - - - - - - - - + + + @@ -1912,6 +1845,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -2032,11 +1986,6 @@ - - - - - @@ -2660,11 +2609,6 @@ - - - - - @@ -2680,9 +2624,24 @@ - - - + + + + + + + + + + + + + + + + + + @@ -4063,14 +4022,14 @@ - - - + + + - - - + + + @@ -4353,31 +4312,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index a4b76b9530d66..9bbc975c742b2 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e712035eabc7b..2a6e21b2ba89a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip +distributionSha256Sum=fba8464465835e74f7270bbf43d6d8a8d7709ab0a43ce1aa3323f73e9aa0c612 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index f3b75f3b0d4fa..faf93008b77e7 100755 --- a/gradlew +++ b/gradlew @@ -205,7 +205,7 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle index d5842d4a2c59c..d5b1bd6ac648e 100644 --- a/libs/cli/build.gradle +++ b/libs/cli/build.gradle @@ -12,6 +12,7 @@ apply plugin: 'elasticsearch.publish' dependencies { api 'net.sf.jopt-simple:jopt-simple:5.0.2' api project(':libs:core') + api project(':libs:logging') testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'cli' diff --git a/libs/cli/src/main/java/module-info.java b/libs/cli/src/main/java/module-info.java index e3969a1c74375..eeaf2eae06b96 100644 --- a/libs/cli/src/main/java/module-info.java +++ b/libs/cli/src/main/java/module-info.java @@ -11,6 +11,8 @@ module org.elasticsearch.cli { requires jopt.simple; requires org.elasticsearch.base; + requires java.logging; + requires org.elasticsearch.logging; exports org.elasticsearch.cli; } diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java index 6d38408ed165a..1690515532e7b 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java @@ -15,6 +15,8 @@ import joptsimple.OptionSpec; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.logging.Level; +import org.elasticsearch.logging.internal.spi.LoggerFactory; import java.io.Closeable; import java.io.IOException; @@ -84,12 +86,16 @@ protected void mainWithoutErrorHandling(String[] args, Terminal terminal, Proces return; } + LoggerFactory loggerFactory = LoggerFactory.provider(); if (options.has(silentOption)) { terminal.setVerbosity(Terminal.Verbosity.SILENT); + loggerFactory.setRootLevel(Level.OFF); } else if (options.has(verboseOption)) { terminal.setVerbosity(Terminal.Verbosity.VERBOSE); + loggerFactory.setRootLevel(Level.DEBUG); } else { terminal.setVerbosity(Terminal.Verbosity.NORMAL); + loggerFactory.setRootLevel(Level.INFO); } execute(terminal, options, processInfo); diff --git a/libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java b/libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java new file mode 100644 index 0000000000000..5d3831881f285 --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.core; + +/** + * A {@link java.util.function.Supplier}-like interface which allows throwing checked exceptions. + */ +@FunctionalInterface +public interface CheckedSupplier { + T get() throws E; +} diff --git a/libs/entitlement/README.md b/libs/entitlement/README.md index 2ab76cf1c2221..893499aaaae4a 100644 --- a/libs/entitlement/README.md +++ b/libs/entitlement/README.md @@ -1,11 +1,192 @@ -### Entitlement library +# Entitlements -This module implements mechanisms to grant and check permissions under the _entitlements_ system. +This module implements mechanisms to grant and check permissions under the _Entitlements_ system. -The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. -The `entitlement-agent` instruments sensitive class library methods with calls to this module, in order to enforce the controls. +The entitlements system provides an alternative to the legacy Java Security Manager; +Elasticsearch (ES) has previously made heavy use of the Java Security Manager to minimize the risk of security vulnerabilities impact. The Java Security Manager has been [deprecated for removal since Java 17](https://openjdk.org/jeps/411) (Sept 2021) and has been [removed in JDK 24](https://openjdk.org/jeps/486) (March 2025). Without an alternative, the removal of the Java Security Manager would have left Elasticsearch users more susceptible to future security vulnerabilities. -This feature is currently under development, and it is completely disabled by default (the agent is not loaded). To enable it, run Elasticsearch with +The goal of _entitlements_ is to protect certain sensitive operations on resources, and the JVM itself, from unexpected and unwanted access, e.g. to limit the scope of potential remote code execution (RCE) vulnerabilities. + +In practice, an entitlement allows code to call a well-defined set of corresponding JDK methods; without the entitlement code calling into those JDK methods is blocked and gets a `NotEntitledException`. + +## Structure + +All the code implementing Entitlements can be found under this directory. The `agent` module bootstraps the entitlement lib, and uses it to instruments sensitive JDK class library methods using a `InstrumentationService`. The current implementation of the instrumentation service uses ASM and is located under `asm-provider`. + +`InstrumentationService` transform JDK methods to start with a call to check entitlements. The entitlement checker is defined in the `bridge`, which is patched into `java.base` at runtime because it must exist in the platform classloader. + +The entitlement checker is implemented in the entitlement lib, which the `bridge` grabs reflectively. `PolicyManager` is where most checks are actually done. The entitlement lib also contains the implementation of the data objects used to define Entitlements (`Policy`, `Scope` and all classes implementing the `Entitlement` interface) as well as the logic for handling them (`PolicyParser`, `PolicyUtils`). + +![Alt text](./entitlements-loading.svg) + +## Policies + +A `Policy` is associated with a single `component` (i.e. Elasticsearch module/plugin or server) and represents the entitlements allowed for a particular `Scope` (i.e. Java module). + +Entitlements are divided into 3 categories: +- available everywhere (Elasticsearch module/plugin or server) +- available only to Elasticsearch modules +- not externally available: can be used only to specify entitlements for modules in the server layer. + +In order to help developers adding the correct entitlements to a policy, the name of the component, the scope name (Java module) and the name of the missing entitlement are specified in the `NotEntitledException` message: +``` +NotEntitledException: component [(server)], module [org.apache.lucene.misc], class [class org.apache.lucene.misc.store.DirectIODirectory], entitlement [read_store_attributes] +``` + +### How to add an Elasticsearch module/plugin policy + +A policy is defined in an `entitlements-policy.yaml` file within an Elasticsearch module/plugin under `src/main/plugin-metadata`. Policy files contain lists of entitlements that should be allowed, grouped by Java module name, which acts as the policy scope. For example, the `transport-netty4` Elasticsearch module's policy file contains an entitlement to accept `inbound_network` connections, limited to the `io.netty.transport` and `io.netty.common` Java modules. + +Elasticsearch modules/plugins that are not yet modularized (i.e. do not have `module-info.java`) will need to use single `ALL-UNNAMED` scope. For example, the `reindex` Elasticsearch module's policy file contains a single `ALL-UNNAMED` scope, with an entitlement to perform `outbound_network`; all code in `reindex` will be able to connect to the network. It is not possible to use the `ALL-UNNAMED` scope for modularized modules/plugins. + +How to add an Entitlements plugin policy is described in the official Elasticsearch docs on how to [create a classic plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/creating-classic-plugins.html). The list of entitlements available to plugins is also described there. + +For Elasticsearch modules, the process is the same. In addition to the entitlements available for plugins, Elasticsearch modules can specify the additional entitlements: + +#### `create_class_loader` +Allows code to construct a Java ClassLoader. + +#### `write_all_system_properties` +This entitlement is similar to `write_system_properties`, but it's not necessary to specify the property names that code in the scope can write: all properties can be written by code with this entitlement. + +#### `inbound_network` +This entitlement is currently available to plugins too; however, we plan to make it internally available only as soon as we can. It will remain available to Elasticsearch modules. + +### How to add a server layer entitlement + +Entitlements for modules in the server layer are grouped in a "server policy"; this policy is builtin into Elasticsearch, expressed in Java code in `EntitlementInitialization` (see `EntitlementInitialization#createPolicyManager`). As such, it can use entitlements that are not externally available, namely `ReadStoreAttributesEntitlement` and `ExitVMEntitlement`. + +In order to add an entitlement, first look if the scope is already present in the server policy. If it's not present, add one. If it is, add an instance of the correct entitlement class to the list of entitlements for that scope. +There is a direct mapping between the entitlement name and the Entitlement class: the name is written in snake case (e.g. `example_name`), the corresponding class has the same name but in Pascal Case with the addition of a `Entitlement` suffix (e.g. `ExampleNameEntitlement`). + +For example, to fix the `NotEntitledException` from the example above: +```java +new Scope( + "org.apache.lucene.misc", + List.of( + new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))), + new ReadStoreAttributesEntitlement() // <- add this new entitlement + ) +) +``` +In any case, before adding a `server` entitlement or make any change to the server layer policy, please consult with the Core/Infra team. + +### Always denied + +Finally, there are some actions that are always denied; these actions do not have an associated entitlement, they are blocked with no option to allow them via a policy. Examples are: spawning a new process, manipulating files via file descriptors, starting a HTTP server, changing the locale, timezone, in/out/err streams, the default exception handler, etc. + +## Tips + +### What to do when you have a NotEntitledException + +You can realize the code you are developing bumps into a `NotEntitledException`; that means your code (or code you are referencing from a 3rd party library) is attempting to perform a sensitive action, and it does not have an entitlement for that, so we are blocking it. + +A `NotEntitledException` could be handled by your code/by your library (via a try-catch, usually of `SecurityException`); in that case, you will still see a WARN log for the "Not entitled" call. + +To distinguish these two cases, check the stack trace of the `NotEntitledException` and look for a frame that catches the exception. + +If you find such a frame, then this `NotEntitledException` could be benign: the code knows how to handle a `SecurityException` and continue with an alternative strategy. In this case, the remedy is probably to suppress the warning. + +If you do not find such a frame, then the remedy is to grant the entitlement. +If the entitlement should or could not be granted (e.g. because the offending code is trying to start a process), then you need to modify your code so it does not perform the sensitive (and forbidden) operation anymore. + +#### Suppress a benign warning + +For a benign `NotEntitledException` that is caught, we probably want to ignore the warning. +Double-check with Core/Infra before embarking on this, because it's a nontrivial amount of work, which goes beyond changes to Elasticsearch and may involve multiple PRs in multiple repos. We want to make sure this is the way to go before spending time on it. + +Suppressing the warning involves adding a setting to the `log4j2.properties` files; you can follow [this PR](https://github.com/elastic/elasticsearch/pull/124883) as an example. Use a consistent naming convention, e.g. `logger.entitlements_.name`. Avoid using extra dots, use `_` instead. + +Each component has its own `log4j2.properties` file. Place the file in `src/main/config`: the build process will take care of bundling the file. + +#### Patching a policy via system properties + +In an emergency, policies for Elasticsearch modules and plugins, and for the server layer modules, can be patched via a system property. +The system property is in the form `es.entitlements.policy.` (`es.entitlements.policy.server` for the server layer policy), and accepts a versioned policy: +```yaml +versions: + - version1 + - versionN +policy: + +``` +For example: +```yaml +versions: + - 9.1.0 +policy: + ALL-UNNAMED: + - set_https_connection_properties + - outbound_network + - files: + - relative_path: ".config/gcloud" + relative_to: home + mode: read +``` + +The versioned policy needs to be base64 encoded, e.g. by placing the policy in a file like `plugin-patch.yaml` and the `base64` command line tool which is included in many OSes: ```shell -./gradlew run --entitlements +base64 -i plugin-patch.yaml +``` +The base64 string will then need to be passed via the command line to ES. +For example, to pass the above policy to a test cluster via gradle run: +```shell +./gradlew run --debug-jvm -Dtests.jvm.argline="-Des.entitlements.policy.repository-gcs=dmVyc2lvbnM6CiAgLSA5LjEuMApwb2xpY3k6CiAgQUxMLVVOTkFNRUQ6CiAgICAtIHNldF9odHRwc19jb25uZWN0aW9uX3Byb3BlcnRpZXMKICAgIC0gb3V0Ym91bmRfbmV0d29yawogICAgLSBmaWxlczoKICAgICAgLSByZWxhdGl2ZV9wYXRoOiAiLmNvbmZpZy9nY2xvdWQiCiAgICAgICAgcmVsYXRpdmVfdG86IGhvbWUKICAgICAgICBtb2RlOiByZWFkCg==" +``` +The versions listed in the policy are string-matched against the Elasticsearch version as returned by `Build.version().current()`. It is possible to specify any number of versions. + +The patch policy will be merged into the existing policy; in other words, entitlements specified in the patch policy will be **added** to the existing policy. + +For example, if you add an entitlement to an existing scope: +```yaml +versions: + - 9.1.0 +policy: + java.desktop: + - manage_threads +``` +with base64 +``` +dmVyc2lvbnM6CiAgLSA5LjEuMApwb2xpY3k6CiAgamF2YS5kZXNrdG9wOgogICAgLSBtYW5hZ2VfdGhyZWFkcw== +``` +That policy is parsed and used to patch the existing entitlements to `java.desktop`, so at the end that module will have both the `load_native_libraries` (from the server layer embedded policy) and the `manage_threads` entitlements (from the patch). + +It is also possible to modify a current entitlement; for `files` and `write_system_properties`, this means that the 2 entitlements (from the patch and from the embedded policy) will be **merged**, taking fields from both of them, so you can grant access to additional files, upgrade access to `read_write`, or add a system property. +You can also add an entitlement to a new scope. It is not possible to remove an entitlement or a scope, or remove fields from an entitlement (e.g. remove access to a path or downgrade access to read-only). + +If the policy is parsed and applied correctly, a INFO log will be displayed: +``` +[INFO ][o.e.e.r.p.PolicyUtils ] [runTask-0] Using policy patch for layer [server] ``` +If you try to add an invalid policy (syntax error, wrong scope, etc.) the patch will be discarded and Elasticsearch will run with the embedded policy. Same if the version does not match. In that case, you’ll see a WARN log: +``` +[WARN ][o.e.e.r.p.PolicyUtils ] [runTask-0] Found a policy patch with invalid content. The patch will not be applied. Layer [server] +java.lang.IllegalStateException: Invalid module name in policy: layer [server] does not have module [java.xml]; available modules [...]; policy path [] +``` + +IMPORTANT: this patching mechanism is intended to be used **only** for emergencies; once a missing entitlement is identified, the fix needs to be applied to the codebase, by raising a PR or submitting a bug via Github so that the bundled policies can be fixed. + +### How to migrate a from a Java Security Manager Policy to an entitlement policy + +Translating Java Security Permissions to Entitlements is usually not too difficult; +- many permissions are not used anymore. The Entitlement system is targeting sensitive actions we identified as crucial to our code; any other permission is not checked anymore. Also, we do not have any entitlement related to reflection or access checks: Elasticsearch runs modularized, and we leverage and trust the Java module mechanism to enforce access and visibility. +Examples of permissions that do not have an Entitlement equivalent: + - `java.net.NetPermission "getProxySelector"`, `java.util.PropertyPermission "", "read"` or `java.lang.RuntimePermission "getClassLoader"`: we do not care about anything that "reads" or "gets" something. We care about writing or setting (except network and files); + - `javax.security.auth.*Permission` or `java.security.SecurityPermission`: currently, we do not have any equivalent to authn/authz permissions. This could change in a future release. + - `java.lang.reflect.ReflectPermission "suppressAccessChecks";` or `java.lang.RuntimePermission "accessDeclaredMembers"`: we rely on Java module encapsulation to protect sensitive classes and methods. + - `java.net.SocketPermission "*", "resolve"` +- some permissions have a 1-1 translation. Examples: + - `java.net.SocketPermission "*", "connect"` translates to `outgoing_network` + - `java.net.SocketPermission "*", "accept"` or `listen` translates to `incoming_network` + - `java.lang.RuntimePermission "createClassLoader"` translates to `create_class_loader` + - `java.io.FilePermission` translates to `files` + - `java.util.PropertyPermission "", "write"` translates to `write_system_properties` (`write_all_system_properties` in case `` is `"*"`) + - `java.lang.RuntimePermission "setContextClassLoader"` translates to `manage_threads` + - `java.lang.RuntimePermission "loadLibrary*"` translates to `load_native_libraries` +- some permissions need more investigation: + - `java.lang.RuntimePermission "setFactory"`: most of the methods that used to be guarded by this permission are always denied; some are always granted. The only equivalent in the entitlement system is `set_https_connection_properties`, for methods like `HttpsURLConnection.setSSLSocketFactory` that can be used to change a HTTPS connection properties after the connection object has been created. + +Note however that there is a key difference in the policy check model between Security Manager and Entitlements. This means that translating a Security Manager policy to an Entitlement policy may not be a 1-1 mapping from Permissions to Entitlements. Security Manager used to do a full-stack check, truncated by `doPrivileged` blocks; Entitlements check the "first untrusted frame". This means that some Permissions that needed to be granted with Security Manager may not need the equivalent entitlement; conversely, code that used `doPrivileged` under the Security Manager model might have not needed a Permission, but might need an Entitlement now to run correctly. + +Finally, a word on scopes: the Security Manager model used either general grants, or granted some permission to a specific codebase, e.g. `grant codeBase "${codebase.netty-transport}"`. +In Entitlements, there is no option for a general grant: you must identify to which module a particular entitlement needs to be granted (except for non-modular plugins, for which everything falls under `ALL-UNNAMED`). If the Security Manager policy specified a codebase, it's usually easy to find the correct module, otherwise it might be tricky and require deeper investigation. diff --git a/libs/entitlement/asm-provider/build.gradle b/libs/entitlement/asm-provider/build.gradle index dcec0579a5bae..d992792cd96d8 100644 --- a/libs/entitlement/asm-provider/build.gradle +++ b/libs/entitlement/asm-provider/build.gradle @@ -11,10 +11,18 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':libs:entitlement') + compileOnly project(':libs:core') + compileOnly project(':libs:logging') implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-util:9.7.1' + implementation 'org.ow2.asm:asm-tree:9.7.1' + implementation 'org.ow2.asm:asm-analysis:9.7.1' testImplementation project(":test:framework") testImplementation project(":libs:entitlement:bridge") - testImplementation 'org.ow2.asm:asm-util:9.7.1' +} + +tasks.named("dependencyLicenses").configure { + mapping from: /asm-.*/, to: 'asm' } tasks.named('test').configure { diff --git a/libs/entitlement/asm-provider/src/main/java/module-info.java b/libs/entitlement/asm-provider/src/main/java/module-info.java index 8cbeafc9013aa..ed75bc2136f34 100644 --- a/libs/entitlement/asm-provider/src/main/java/module-info.java +++ b/libs/entitlement/asm-provider/src/main/java/module-info.java @@ -12,7 +12,11 @@ module org.elasticsearch.entitlement.instrumentation { requires org.objectweb.asm; + requires org.objectweb.asm.util; requires org.elasticsearch.entitlement; + requires static org.elasticsearch.base; // for SuppressForbidden + requires org.elasticsearch.logging; + provides InstrumentationService with InstrumentationServiceImpl; } diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java index eaf4d0ad98ef5..ffcc23e16d1f6 100644 --- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java @@ -9,6 +9,7 @@ package org.elasticsearch.entitlement.instrumentation.impl; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.instrumentation.CheckMethod; import org.elasticsearch.entitlement.instrumentation.InstrumentationService; import org.elasticsearch.entitlement.instrumentation.Instrumenter; @@ -20,48 +21,208 @@ import org.objectweb.asm.Type; import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayDeque; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class InstrumentationServiceImpl implements InstrumentationService { + private static final String OBJECT_INTERNAL_NAME = Type.getInternalName(Object.class); + @Override public Instrumenter newInstrumenter(Class clazz, Map methods) { return InstrumenterImpl.create(clazz, methods); } + private interface CheckerMethodVisitor { + void visit(Class currentClass, int access, String checkerMethodName, String checkerMethodDescriptor); + } + + private void visitClassAndSupers(Class checkerClass, CheckerMethodVisitor checkerMethodVisitor) throws ClassNotFoundException { + Set> visitedClasses = new HashSet<>(); + ArrayDeque> classesToVisit = new ArrayDeque<>(Collections.singleton(checkerClass)); + while (classesToVisit.isEmpty() == false) { + var currentClass = classesToVisit.remove(); + if (visitedClasses.contains(currentClass)) { + continue; + } + visitedClasses.add(currentClass); + + try { + var classFileInfo = InstrumenterImpl.getClassFileInfo(currentClass); + ClassReader reader = new ClassReader(classFileInfo.bytecodes()); + ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + try { + if (OBJECT_INTERNAL_NAME.equals(superName) == false) { + classesToVisit.add(Class.forName(Type.getObjectType(superName).getClassName())); + } + for (var interfaceName : interfaces) { + classesToVisit.add(Class.forName(Type.getObjectType(interfaceName).getClassName())); + } + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Cannot inspect checker class " + currentClass.getName(), e); + } + } + + @Override + public MethodVisitor visitMethod( + int access, + String checkerMethodName, + String checkerMethodDescriptor, + String signature, + String[] exceptions + ) { + var mv = super.visitMethod(access, checkerMethodName, checkerMethodDescriptor, signature, exceptions); + checkerMethodVisitor.visit(currentClass, access, checkerMethodName, checkerMethodDescriptor); + return mv; + } + }; + reader.accept(visitor, 0); + } catch (IOException e) { + throw new ClassNotFoundException("Cannot find a definition for class [" + checkerClass.getName() + "]", e); + } + } + } + @Override - public Map lookupMethods(Class checkerClass) throws IOException { - var methodsToInstrument = new HashMap(); - var classFileInfo = InstrumenterImpl.getClassFileInfo(checkerClass); - ClassReader reader = new ClassReader(classFileInfo.bytecodes()); - ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { - @Override - public MethodVisitor visitMethod( - int access, - String checkerMethodName, - String checkerMethodDescriptor, - String signature, - String[] exceptions - ) { - var mv = super.visitMethod(access, checkerMethodName, checkerMethodDescriptor, signature, exceptions); + public Map lookupMethods(Class checkerClass) throws ClassNotFoundException { + Map methodsToInstrument = new HashMap<>(); + visitClassAndSupers(checkerClass, (currentClass, access, checkerMethodName, checkerMethodDescriptor) -> { + if (checkerMethodName.startsWith(InstrumentationService.CHECK_METHOD_PREFIX)) { var checkerMethodArgumentTypes = Type.getArgumentTypes(checkerMethodDescriptor); var methodToInstrument = parseCheckerMethodSignature(checkerMethodName, checkerMethodArgumentTypes); var checkerParameterDescriptors = Arrays.stream(checkerMethodArgumentTypes).map(Type::getDescriptor).toList(); - var checkMethod = new CheckMethod(Type.getInternalName(checkerClass), checkerMethodName, checkerParameterDescriptors); + var checkMethod = new CheckMethod(Type.getInternalName(currentClass), checkerMethodName, checkerParameterDescriptors); + methodsToInstrument.putIfAbsent(methodToInstrument, checkMethod); + } + }); + + return methodsToInstrument; + } - methodsToInstrument.put(methodToInstrument, checkMethod); + @SuppressForbidden(reason = "Need access to abstract methods (protected/package internal) in base class") + @Override + public InstrumentationInfo lookupImplementationMethod( + Class targetSuperclass, + String targetMethodName, + Class implementationClass, + Class checkerClass, + String checkMethodName, + Class... parameterTypes + ) throws NoSuchMethodException, ClassNotFoundException { + + var targetMethod = targetSuperclass.getDeclaredMethod(targetMethodName, parameterTypes); + var implementationMethod = implementationClass.getMethod(targetMethod.getName(), targetMethod.getParameterTypes()); + validateTargetMethod(implementationClass, targetMethod, implementationMethod); + + var checkerAdditionalArguments = Stream.of(Class.class, targetSuperclass); + var checkMethodArgumentTypes = Stream.concat(checkerAdditionalArguments, Arrays.stream(parameterTypes)) + .map(Type::getType) + .toArray(Type[]::new); - return mv; + CheckMethod[] checkMethod = new CheckMethod[1]; + + visitClassAndSupers(checkerClass, (currentClass, access, methodName, methodDescriptor) -> { + if (methodName.equals(checkMethodName)) { + var methodArgumentTypes = Type.getArgumentTypes(methodDescriptor); + if (Arrays.equals(methodArgumentTypes, checkMethodArgumentTypes)) { + var checkerParameterDescriptors = Arrays.stream(methodArgumentTypes).map(Type::getDescriptor).toList(); + checkMethod[0] = new CheckMethod(Type.getInternalName(currentClass), methodName, checkerParameterDescriptors); + } } - }; - reader.accept(visitor, 0); - return methodsToInstrument; + }); + + if (checkMethod[0] == null) { + throw new NoSuchMethodException( + String.format( + Locale.ROOT, + "Cannot find a method with name [%s] and arguments [%s] in class [%s]", + checkMethodName, + Arrays.stream(checkMethodArgumentTypes).map(Type::toString).collect(Collectors.joining()), + checkerClass.getName() + ) + ); + } + + return new InstrumentationInfo( + new MethodKey( + Type.getInternalName(implementationMethod.getDeclaringClass()), + implementationMethod.getName(), + Arrays.stream(parameterTypes).map(c -> Type.getType(c).getInternalName()).toList() + ), + checkMethod[0] + ); + } + + private static void validateTargetMethod(Class implementationClass, Method targetMethod, Method implementationMethod) { + if (targetMethod.getDeclaringClass().isAssignableFrom(implementationClass) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Not an implementation class for %s: %s does not implement %s", + targetMethod.getName(), + implementationClass.getName(), + targetMethod.getDeclaringClass().getName() + ) + ); + } + if (Modifier.isPrivate(targetMethod.getModifiers())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Not a valid instrumentation method: %s is private in %s", + targetMethod.getName(), + targetMethod.getDeclaringClass().getName() + ) + ); + } + if (Modifier.isStatic(targetMethod.getModifiers())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Not a valid instrumentation method: %s is static in %s", + targetMethod.getName(), + targetMethod.getDeclaringClass().getName() + ) + ); + } + var methodModifiers = implementationMethod.getModifiers(); + if (Modifier.isAbstract(methodModifiers)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Not a valid instrumentation method: %s is abstract in %s", + targetMethod.getName(), + implementationClass.getName() + ) + ); + } + if (Modifier.isPublic(methodModifiers) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Not a valid instrumentation method: %s is not public in %s", + targetMethod.getName(), + implementationClass.getName() + ) + ); + } } private static final Type CLASS_TYPE = Type.getType(Class.class); @@ -85,8 +246,8 @@ static ParsedCheckerMethod parseCheckerMethodName(String checkerMethodName) { String.format( Locale.ROOT, "Checker method %s has incorrect name format. " - + "It should be either check$$methodName (instance), check$package_ClassName$methodName (static) or " - + "check$package_ClassName$ (ctor)", + + "It should be either check$package_ClassName$methodName (instance), check$package_ClassName$$methodName (static) " + + "or check$package_ClassName$ (ctor)", checkerMethodName ) ); diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java index 06408941ac96e..ed13f6d67014b 100644 --- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java @@ -9,9 +9,13 @@ package org.elasticsearch.entitlement.instrumentation.impl; +import org.elasticsearch.core.Strings; import org.elasticsearch.entitlement.instrumentation.CheckMethod; +import org.elasticsearch.entitlement.instrumentation.EntitlementInstrumented; import org.elasticsearch.entitlement.instrumentation.Instrumenter; import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; @@ -21,21 +25,23 @@ import org.objectweb.asm.Opcodes; import org.objectweb.asm.RecordComponentVisitor; import org.objectweb.asm.Type; +import org.objectweb.asm.util.CheckClassAdapter; import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.Map; import java.util.stream.Stream; import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; import static org.objectweb.asm.Opcodes.ACC_STATIC; -import static org.objectweb.asm.Opcodes.GETSTATIC; import static org.objectweb.asm.Opcodes.INVOKEINTERFACE; import static org.objectweb.asm.Opcodes.INVOKESTATIC; -import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL; public class InstrumenterImpl implements Instrumenter { + private static final Logger logger = LogManager.getLogger(InstrumenterImpl.class); private final String getCheckerClassMethodDescriptor; private final String handleClass; @@ -59,6 +65,7 @@ public class InstrumenterImpl implements Instrumenter { } public static InstrumenterImpl create(Class checkerClass, Map checkMethods) { + Type checkerClassType = Type.getType(checkerClass); String handleClass = checkerClassType.getInternalName() + "Handle"; String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(checkerClassType); @@ -78,18 +85,59 @@ static ClassFileInfo getClassFileInfo(Class clazz) throws IOException { return new ClassFileInfo(fileName, originalBytecodes); } + private enum VerificationPhase { + BEFORE_INSTRUMENTATION, + AFTER_INSTRUMENTATION + } + + private static String verify(byte[] classfileBuffer) { + ClassReader reader = new ClassReader(classfileBuffer); + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter); + CheckClassAdapter.verify(reader, false, printWriter); + return stringWriter.toString(); + } + + private static void verifyAndLog(byte[] classfileBuffer, String className, VerificationPhase phase) { + try { + String result = verify(classfileBuffer); + if (result.isEmpty() == false) { + logger.error(Strings.format("Bytecode verification (%s) for class [%s] failed: %s", phase, className, result)); + } else { + logger.info("Bytecode verification ({}) for class [{}] passed", phase, className); + } + } catch (ClassCircularityError e) { + // Apparently, verification during instrumentation is challenging for class resolution and loading + // Treat this not as an error, but as "inconclusive" + logger.warn(Strings.format("Cannot perform bytecode verification (%s) for class [%s]", phase, className), e); + } catch (IllegalArgumentException e) { + // The ASM CheckClassAdapter in some cases throws this instead of printing the error + logger.error(Strings.format("Bytecode verification (%s) for class [%s] failed", phase, className), e); + } + } + @Override - public byte[] instrumentClass(String className, byte[] classfileBuffer) { + public byte[] instrumentClass(String className, byte[] classfileBuffer, boolean verify) { + if (verify) { + verifyAndLog(classfileBuffer, className, VerificationPhase.BEFORE_INSTRUMENTATION); + } + ClassReader reader = new ClassReader(classfileBuffer); ClassWriter writer = new ClassWriter(reader, COMPUTE_FRAMES | COMPUTE_MAXS); ClassVisitor visitor = new EntitlementClassVisitor(Opcodes.ASM9, writer, className); reader.accept(visitor, 0); - return writer.toByteArray(); + var outBytes = writer.toByteArray(); + + if (verify) { + verifyAndLog(outBytes, className, VerificationPhase.AFTER_INSTRUMENTATION); + } + + return outBytes; } class EntitlementClassVisitor extends ClassVisitor { - private static final String ENTITLEMENT_ANNOTATION = "EntitlementInstrumented"; + private static final String ENTITLEMENT_ANNOTATION_DESCRIPTOR = Type.getDescriptor(EntitlementInstrumented.class); private final String className; @@ -108,7 +156,7 @@ public void visit(int version, int access, String name, String signature, String @Override public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) { - if (visible && descriptor.equals(ENTITLEMENT_ANNOTATION)) { + if (visible && descriptor.equals(ENTITLEMENT_ANNOTATION_DESCRIPTOR)) { isAnnotationPresent = true; annotationNeeded = false; } @@ -152,14 +200,13 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, Str if (isAnnotationPresent == false) { boolean isStatic = (access & ACC_STATIC) != 0; boolean isCtor = "".equals(name); - boolean hasReceiver = (isStatic || isCtor) == false; var key = new MethodKey(className, name, Stream.of(Type.getArgumentTypes(descriptor)).map(Type::getInternalName).toList()); var instrumentationMethod = checkMethods.get(key); if (instrumentationMethod != null) { - // LOGGER.debug("Will instrument method {}", key); + logger.debug("Will instrument {}", key); return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, isCtor, descriptor, instrumentationMethod); } else { - // LOGGER.trace("Will not instrument method {}", key); + logger.trace("Will not instrument {}", key); } } return mv; @@ -175,7 +222,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, Str private void addClassAnnotationIfNeeded() { if (annotationNeeded) { // logger.debug("Adding {} annotation", ENTITLEMENT_ANNOTATION); - AnnotationVisitor av = cv.visitAnnotation(ENTITLEMENT_ANNOTATION, true); + AnnotationVisitor av = cv.visitAnnotation(ENTITLEMENT_ANNOTATION_DESCRIPTOR, true); if (av != null) { av.visitEnd(); } @@ -237,22 +284,9 @@ private void pushCallerClass() { false ); } else { - mv.visitFieldInsn( - GETSTATIC, - Type.getInternalName(StackWalker.Option.class), - "RETAIN_CLASS_REFERENCE", - Type.getDescriptor(StackWalker.Option.class) - ); mv.visitMethodInsn( INVOKESTATIC, - Type.getInternalName(StackWalker.class), - "getInstance", - Type.getMethodDescriptor(Type.getType(StackWalker.class), Type.getType(StackWalker.Option.class)), - false - ); - mv.visitMethodInsn( - INVOKEVIRTUAL, - Type.getInternalName(StackWalker.class), + "org/elasticsearch/entitlement/bridge/Util", "getCallerClass", Type.getMethodDescriptor(Type.getType(Class.class)), false diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java index ab0d96a8df96d..b278605c7fe73 100644 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.test.ESTestCase; import org.objectweb.asm.Type; -import java.io.IOException; import java.util.List; import java.util.Map; @@ -29,7 +28,30 @@ public class InstrumentationServiceImplTests extends ESTestCase { final InstrumentationService instrumentationService = new InstrumentationServiceImpl(); - static class TestTargetClass {} + interface TestTargetInterface { + void instanceMethod(int x, String y); + } + + static class TestTargetClass implements TestTargetInterface { + @Override + public void instanceMethod(int x, String y) {} + } + + abstract static class TestTargetBaseClass { + abstract void instanceMethod(int x, String y); + + abstract void instanceMethod2(int x, String y); + } + + abstract static class TestTargetIntermediateClass extends TestTargetBaseClass { + @Override + public void instanceMethod2(int x, String y) {} + } + + static class TestTargetImplementationClass extends TestTargetIntermediateClass { + @Override + public void instanceMethod(int x, String y) {} + } interface TestChecker { void check$org_example_TestTargetClass$$staticMethod(Class clazz, int arg0, String arg1, Object arg2); @@ -39,6 +61,14 @@ interface TestChecker { void check$org_example_TestTargetClass$instanceMethodWithArgs(Class clazz, TestTargetClass that, int x, int y); } + interface TestCheckerDerived extends TestChecker { + void check$org_example_TestTargetClass$instanceMethodNoArgs(Class clazz, TestTargetClass that); + + void check$org_example_TestTargetClass$differentInstanceMethod(Class clazz, TestTargetClass that); + } + + interface TestCheckerDerived2 extends TestCheckerDerived, TestChecker {} + interface TestCheckerOverloads { void check$org_example_TestTargetClass$$staticMethodWithOverload(Class clazz, int x, int y); @@ -51,7 +81,15 @@ interface TestCheckerCtors { void check$org_example_TestTargetClass$(Class clazz, int x, String y); } - public void testInstrumentationTargetLookup() throws IOException { + interface TestCheckerMixed { + void check$org_example_TestTargetClass$$staticMethod(Class clazz, int arg0, String arg1, Object arg2); + + void checkInstanceMethodManual(Class clazz, TestTargetInterface that, int x, String y); + + void checkInstanceMethodManual(Class clazz, TestTargetBaseClass that, int x, String y); + } + + public void testInstrumentationTargetLookup() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestChecker.class); assertThat(checkMethods, aMapWithSize(3)); @@ -104,7 +142,7 @@ public void testInstrumentationTargetLookup() throws IOException { ); } - public void testInstrumentationTargetLookupWithOverloads() throws IOException { + public void testInstrumentationTargetLookupWithOverloads() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestCheckerOverloads.class); assertThat(checkMethods, aMapWithSize(2)); @@ -136,7 +174,76 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException { ); } - public void testInstrumentationTargetLookupWithCtors() throws IOException { + public void testInstrumentationTargetLookupWithDerivedClass() throws ClassNotFoundException { + Map checkMethods = instrumentationService.lookupMethods(TestCheckerDerived2.class); + + assertThat(checkMethods, aMapWithSize(4)); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "staticMethod", List.of("I", "java/lang/String", "java/lang/Object"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker", + "check$org_example_TestTargetClass$$staticMethod", + List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;", "Ljava/lang/Object;") + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "instanceMethodNoArgs", List.of())), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerDerived", + "check$org_example_TestTargetClass$instanceMethodNoArgs", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;" + ) + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "instanceMethodWithArgs", List.of("I", "I"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker", + "check$org_example_TestTargetClass$instanceMethodWithArgs", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;", + "I", + "I" + ) + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "differentInstanceMethod", List.of())), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerDerived", + "check$org_example_TestTargetClass$differentInstanceMethod", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;" + ) + ) + ) + ) + ); + } + + public void testInstrumentationTargetLookupWithCtors() throws ClassNotFoundException { Map checkMethods = instrumentationService.lookupMethods(TestCheckerCtors.class); assertThat(checkMethods, aMapWithSize(2)); @@ -168,6 +275,139 @@ public void testInstrumentationTargetLookupWithCtors() throws IOException { ); } + public void testInstrumentationTargetLookupWithExtraMethods() throws ClassNotFoundException { + Map checkMethods = instrumentationService.lookupMethods(TestCheckerMixed.class); + + assertThat(checkMethods, aMapWithSize(1)); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "staticMethod", List.of("I", "java/lang/String", "java/lang/Object"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerMixed", + "check$org_example_TestTargetClass$$staticMethod", + List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;", "Ljava/lang/Object;") + ) + ) + ) + ); + } + + public void testLookupImplementationMethodWithInterface() throws ClassNotFoundException, NoSuchMethodException { + var info = instrumentationService.lookupImplementationMethod( + TestTargetInterface.class, + "instanceMethod", + TestTargetClass.class, + TestCheckerMixed.class, + "checkInstanceMethodManual", + int.class, + String.class + ); + + assertThat( + info.targetMethod(), + equalTo( + new MethodKey( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass", + "instanceMethod", + List.of("I", "java/lang/String") + ) + ) + ); + assertThat( + info.checkMethod(), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerMixed", + "checkInstanceMethodManual", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetInterface;", + "I", + "Ljava/lang/String;" + ) + ) + ) + ); + } + + public void testLookupImplementationMethodWithBaseClass() throws ClassNotFoundException, NoSuchMethodException { + var info = instrumentationService.lookupImplementationMethod( + TestTargetBaseClass.class, + "instanceMethod", + TestTargetImplementationClass.class, + TestCheckerMixed.class, + "checkInstanceMethodManual", + int.class, + String.class + ); + + assertThat( + info.targetMethod(), + equalTo( + new MethodKey( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetImplementationClass", + "instanceMethod", + List.of("I", "java/lang/String") + ) + ) + ); + assertThat( + info.checkMethod(), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerMixed", + "checkInstanceMethodManual", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetBaseClass;", + "I", + "Ljava/lang/String;" + ) + ) + ) + ); + } + + public void testLookupImplementationMethodWithInheritance() throws ClassNotFoundException, NoSuchMethodException { + var info = instrumentationService.lookupImplementationMethod( + TestTargetBaseClass.class, + "instanceMethod2", + TestTargetImplementationClass.class, + TestCheckerMixed.class, + "checkInstanceMethodManual", + int.class, + String.class + ); + + assertThat( + info.targetMethod(), + equalTo( + new MethodKey( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetIntermediateClass", + "instanceMethod2", + List.of("I", "java/lang/String") + ) + ) + ); + assertThat( + info.checkMethod(), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerMixed", + "checkInstanceMethodManual", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetBaseClass;", + "I", + "Ljava/lang/String;" + ) + ) + ) + ); + } + public void testParseCheckerMethodSignatureStaticMethod() { var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( "check$org_example_TestClass$$staticMethod", diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java index 35932969a3541..4242c592b0ceb 100644 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java @@ -226,9 +226,9 @@ public void testNotInstrumentedTwice() throws Exception { var instrumenter = createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod)); var loader1 = instrumentTestClass(instrumenter); - byte[] instrumentedTwiceBytecode = instrumenter.instrumentClass(TestClassToInstrument.class.getName(), loader1.testClassBytes); - logger.trace(() -> Strings.format("Bytecode after 2nd instrumentation:\n%s", bytecode2text(instrumentedTwiceBytecode))); - var loader2 = new TestLoader(TestClassToInstrument.class.getName(), instrumentedTwiceBytecode); + byte[] instrumentedTwiceBytes = instrumenter.instrumentClass(TestClassToInstrument.class.getName(), loader1.testClassBytes, true); + logger.trace(() -> Strings.format("Bytecode after 2nd instrumentation:\n%s", bytecode2text(instrumentedTwiceBytes))); + var loader2 = new TestLoader(TestClassToInstrument.class.getName(), instrumentedTwiceBytes); assertStaticMethodThrows(loader2, targetMethod, 123); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); @@ -306,7 +306,7 @@ private static InstrumenterImpl createInstrumenter(Map metho private static TestLoader instrumentTestClass(InstrumenterImpl instrumenter) throws IOException { var clazz = TestClassToInstrument.class; ClassFileInfo initial = getClassFileInfo(clazz); - byte[] newBytecode = instrumenter.instrumentClass(Type.getInternalName(clazz), initial.bytecodes()); + byte[] newBytecode = instrumenter.instrumentClass(Type.getInternalName(clazz), initial.bytecodes(), true); if (logger.isTraceEnabled()) { logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); } diff --git a/libs/entitlement/bridge/build.gradle b/libs/entitlement/bridge/build.gradle index 69be1e3fce5f9..51bde35e8fdc3 100644 --- a/libs/entitlement/bridge/build.gradle +++ b/libs/entitlement/bridge/build.gradle @@ -14,8 +14,11 @@ apply plugin: 'elasticsearch.mrjar' tasks.named('jar').configure { // guarding for intellij - if (sourceSets.findByName("main23")) { - from sourceSets.main23.output + if (sourceSets.findByName("main19")) { + from sourceSets.main19.output + } + if (sourceSets.findByName("main20")) { + from sourceSets.main20.output } if (sourceSets.findByName("main21")) { from sourceSets.main21.output @@ -23,6 +26,9 @@ tasks.named('jar').configure { if (sourceSets.findByName("main22")) { from sourceSets.main22.output } + if (sourceSets.findByName("main23")) { + from sourceSets.main23.output + } } // The bridge only uses things within the jdk, but the checker @@ -31,3 +37,9 @@ tasks.named('jar').configure { tasks.withType(CheckForbiddenApisTask).configureEach { enabled = false } + +// EntitlementChecker interfaces may contain long URLs pointing to JDK code references on GH, or to JDK documentation, +// and @SuppressWarnings for checkstyle does not work for mrjar projects +tasks.withType(Checkstyle).configureEach { + exclude "**/*EntitlementChecker.java" +} diff --git a/libs/entitlement/bridge/src/main/java/module-info.java b/libs/entitlement/bridge/src/main/java/module-info.java index b9055ec5fbf67..6a85013c1f1f5 100644 --- a/libs/entitlement/bridge/src/main/java/module-info.java +++ b/libs/entitlement/bridge/src/main/java/module-info.java @@ -11,6 +11,8 @@ // At build and run time, the bridge is patched into the java.base module. module org.elasticsearch.entitlement.bridge { requires java.net.http; + requires jdk.net; + requires java.logging; exports org.elasticsearch.entitlement.bridge; } diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 532272a71a2b4..74b5b98713ec3 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -9,8 +9,14 @@ package org.elasticsearch.entitlement.bridge; +import jdk.nio.Channels; + import java.io.File; +import java.io.FileDescriptor; +import java.io.FileFilter; +import java.io.FilenameFilter; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; @@ -28,6 +34,7 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.SocketImplFactory; +import java.net.URI; import java.net.URL; import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; @@ -41,14 +48,40 @@ import java.nio.channels.DatagramChannel; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitor; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.UserPrincipal; +import java.nio.file.spi.FileSystemProvider; +import java.security.KeyStore; +import java.security.Provider; import java.security.cert.CertStoreParameters; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ForkJoinPool; +import java.util.function.BiPredicate; +import java.util.logging.FileHandler; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -71,7 +104,7 @@ public interface EntitlementChecker { /// ///////////////// // - // ClassLoader ctor + // create class loaders // void check$java_lang_ClassLoader$(Class callerClass); @@ -80,22 +113,6 @@ public interface EntitlementChecker { void check$java_lang_ClassLoader$(Class callerClass, String name, ClassLoader parent); - /// ///////////////// - // - // SecureClassLoader ctor - // - - void check$java_security_SecureClassLoader$(Class callerClass); - - void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent); - - void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent); - - /// ///////////////// - // - // URLClassLoader constructors - // - void check$java_net_URLClassLoader$(Class callerClass, URL[] urls); void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent); @@ -106,6 +123,12 @@ public interface EntitlementChecker { void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory); + void check$java_security_SecureClassLoader$(Class callerClass); + + void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent); + + void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent); + /// ///////////////// // // "setFactory" methods @@ -133,6 +156,8 @@ public interface EntitlementChecker { // System Properties and similar // + void check$java_lang_System$$setProperties(Class callerClass, Properties props); + void check$java_lang_System$$setProperty(Class callerClass, String key, String value); void check$java_lang_System$$clearProperty(Class callerClass, String key); @@ -142,33 +167,33 @@ public interface EntitlementChecker { // JVM-wide state changes // - void check$java_lang_System$$setIn(Class callerClass, InputStream in); - - void check$java_lang_System$$setOut(Class callerClass, PrintStream out); + void check$com_sun_tools_jdi_VirtualMachineManagerImpl$$virtualMachineManager(Class callerClass); void check$java_lang_System$$setErr(Class callerClass, PrintStream err); - void check$java_lang_System$$setProperties(Class callerClass, Properties props); + void check$java_lang_System$$setIn(Class callerClass, InputStream in); + + void check$java_lang_System$$setOut(Class callerClass, PrintStream out); void check$java_lang_Runtime$addShutdownHook(Class callerClass, Runtime runtime, Thread hook); void check$java_lang_Runtime$removeShutdownHook(Class callerClass, Runtime runtime, Thread hook); - void check$jdk_tools_jlink_internal_Jlink$(Class callerClass); + void check$java_lang_Thread$$setDefaultUncaughtExceptionHandler(Class callerClass, Thread.UncaughtExceptionHandler ueh); - void check$jdk_tools_jlink_internal_Main$$run(Class callerClass, PrintWriter out, PrintWriter err, String... args); + void check$java_net_DatagramSocket$$setDatagramSocketImplFactory(Class callerClass, DatagramSocketImplFactory fac); - void check$jdk_vm_ci_services_JVMCIServiceLocator$$getProviders(Class callerClass, Class service); + void check$java_net_HttpURLConnection$$setFollowRedirects(Class callerClass, boolean set); - void check$jdk_vm_ci_services_Services$$load(Class callerClass, Class service); + void check$java_net_ServerSocket$$setSocketFactory(Class callerClass, SocketImplFactory fac); - void check$jdk_vm_ci_services_Services$$loadSingle(Class callerClass, Class service, boolean required); + void check$java_net_Socket$$setSocketImplFactory(Class callerClass, SocketImplFactory fac); - void check$com_sun_tools_jdi_VirtualMachineManagerImpl$$virtualMachineManager(Class callerClass); + void check$java_net_URL$$setURLStreamHandlerFactory(Class callerClass, URLStreamHandlerFactory fac); - void check$java_lang_Thread$$setDefaultUncaughtExceptionHandler(Class callerClass, Thread.UncaughtExceptionHandler ueh); + void check$java_net_URLConnection$$setFileNameMap(Class callerClass, FileNameMap map); - void check$java_util_spi_LocaleServiceProvider$(Class callerClass); + void check$java_net_URLConnection$$setContentHandlerFactory(Class callerClass, ContentHandlerFactory fac); void check$java_text_spi_BreakIteratorProvider$(Class callerClass); @@ -190,6 +215,8 @@ public interface EntitlementChecker { void check$java_util_spi_LocaleNameProvider$(Class callerClass); + void check$java_util_spi_LocaleServiceProvider$(Class callerClass); + void check$java_util_spi_TimeZoneNameProvider$(Class callerClass); void check$java_util_logging_LogManager$(Class callerClass); @@ -200,19 +227,17 @@ public interface EntitlementChecker { void check$java_util_TimeZone$$setDefault(Class callerClass, TimeZone zone); - void check$java_net_DatagramSocket$$setDatagramSocketImplFactory(Class callerClass, DatagramSocketImplFactory fac); - - void check$java_net_HttpURLConnection$$setFollowRedirects(Class callerClass, boolean set); + void check$jdk_tools_jlink_internal_Jlink$(Class callerClass); - void check$java_net_ServerSocket$$setSocketFactory(Class callerClass, SocketImplFactory fac); + void check$jdk_tools_jlink_internal_Main$$run(Class callerClass, PrintWriter out, PrintWriter err, String... args); - void check$java_net_Socket$$setSocketImplFactory(Class callerClass, SocketImplFactory fac); + void check$jdk_vm_ci_services_JVMCIServiceLocator$$getProviders(Class callerClass, Class service); - void check$java_net_URL$$setURLStreamHandlerFactory(Class callerClass, URLStreamHandlerFactory fac); + void check$jdk_vm_ci_services_Services$$load(Class callerClass, Class service); - void check$java_net_URLConnection$$setFileNameMap(Class callerClass, FileNameMap map); + void check$jdk_vm_ci_services_Services$$loadSingle(Class callerClass, Class service, boolean required); - void check$java_net_URLConnection$$setContentHandlerFactory(Class callerClass, ContentHandlerFactory fac); + void check$java_nio_charset_spi_CharsetProvider$(Class callerClass); /// ///////////////// // @@ -222,10 +247,6 @@ public interface EntitlementChecker { void check$java_net_ResponseCache$$setDefault(Class callerClass, ResponseCache rc); - void check$java_net_spi_InetAddressResolverProvider$(Class callerClass); - - void check$java_net_spi_URLStreamHandlerProvider$(Class callerClass); - void check$java_net_URL$(Class callerClass, String protocol, String host, int port, String file, URLStreamHandler handler); void check$java_net_URL$(Class callerClass, URL context, String spec, URLStreamHandler handler); @@ -236,14 +257,14 @@ public interface EntitlementChecker { void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr); - void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p); - - void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p); - void check$java_net_DatagramSocket$joinGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); void check$java_net_DatagramSocket$leaveGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p); + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr); void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); @@ -254,6 +275,10 @@ public interface EntitlementChecker { void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl); + void check$java_net_spi_InetAddressResolverProvider$(Class callerClass); + + void check$java_net_spi_URLStreamHandlerProvider$(Class callerClass); + // Binding/connecting ctor void check$java_net_ServerSocket$(Class callerClass, int port); @@ -290,9 +315,210 @@ public interface EntitlementChecker { void check$java_net_Socket$connect(Class callerClass, Socket that, SocketAddress endpoint, int backlog); - // Network miscellanea + // URLConnection (java.net + sun.net.www) + + void check$java_net_URL$openConnection(Class callerClass, java.net.URL that); + void check$java_net_URL$openConnection(Class callerClass, java.net.URL that, Proxy proxy); + void check$java_net_URL$openStream(Class callerClass, java.net.URL that); + + void check$java_net_URL$getContent(Class callerClass, java.net.URL that); + + void check$java_net_URL$getContent(Class callerClass, java.net.URL that, Class[] classes); + + void check$java_net_URLConnection$getContentLength(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getContentLengthLong(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getContentType(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getContentEncoding(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getExpiration(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getDate(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getLastModified(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getHeaderFieldInt(Class callerClass, java.net.URLConnection that, String name, int defaultValue); + + void check$java_net_URLConnection$getHeaderFieldLong(Class callerClass, java.net.URLConnection that, String name, long defaultValue); + + void check$java_net_URLConnection$getHeaderFieldDate(Class callerClass, java.net.URLConnection that, String name, long defaultValue); + + void check$java_net_URLConnection$getContent(Class callerClass, java.net.URLConnection that); + + void check$java_net_URLConnection$getContent(Class callerClass, java.net.URLConnection that, Class[] classes); + + void check$java_net_HttpURLConnection$getResponseCode(Class callerClass, java.net.HttpURLConnection that); + + void check$java_net_HttpURLConnection$getResponseMessage(Class callerClass, java.net.HttpURLConnection that); + + void check$java_net_HttpURLConnection$getHeaderFieldDate( + Class callerClass, + java.net.HttpURLConnection that, + String name, + long defaultValue + ); + + // Using java.net.URLConnection for "that" as sun.net.www.* is not exported + void check$sun_net_www_URLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, String name); + + void check$sun_net_www_URLConnection$getHeaderFields(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_URLConnection$getHeaderFieldKey(Class callerClass, java.net.URLConnection that, int n); + + void check$sun_net_www_URLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, int n); + + void check$sun_net_www_URLConnection$getContentType(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_URLConnection$getContentLength(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_ftp_FtpURLConnection$connect(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_ftp_FtpURLConnection$getInputStream(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_ftp_FtpURLConnection$getOutputStream(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_http_HttpURLConnection$$openConnectionCheckRedirects(Class callerClass, java.net.URLConnection c); + + void check$sun_net_www_protocol_http_HttpURLConnection$connect(Class callerClass, java.net.HttpURLConnection that); + + void check$sun_net_www_protocol_http_HttpURLConnection$getOutputStream(Class callerClass, java.net.HttpURLConnection that); + + void check$sun_net_www_protocol_http_HttpURLConnection$getInputStream(Class callerClass, java.net.HttpURLConnection that); + + void check$sun_net_www_protocol_http_HttpURLConnection$getErrorStream(Class callerClass, java.net.HttpURLConnection that); + + void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderField( + Class callerClass, + java.net.HttpURLConnection that, + String name + ); + + void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderFields(Class callerClass, java.net.HttpURLConnection that); + + void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderField(Class callerClass, java.net.HttpURLConnection that, int n); + + void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderFieldKey(Class callerClass, java.net.HttpURLConnection that, int n); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$connect(Class callerClass, javax.net.ssl.HttpsURLConnection that); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getOutputStream( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getInputStream( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getErrorStream( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderField( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFields( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderField( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + int n + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldKey( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + int n + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getResponseCode( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getResponseMessage( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentLength( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentLengthLong( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentType( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentEncoding( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getExpiration(Class callerClass, javax.net.ssl.HttpsURLConnection that); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getDate(Class callerClass, javax.net.ssl.HttpsURLConnection that); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getLastModified( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldInt( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name, + int defaultValue + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldLong( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name, + long defaultValue + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldDate( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name, + long defaultValue + ); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContent(Class callerClass, javax.net.ssl.HttpsURLConnection that); + + void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContent( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + Class[] classes + ); + + void check$sun_net_www_protocol_https_AbstractDelegateHttpsURLConnection$connect(Class callerClass, java.net.HttpURLConnection that); + + void check$sun_net_www_protocol_mailto_MailToURLConnection$connect(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_mailto_MailToURLConnection$getOutputStream(Class callerClass, java.net.URLConnection that); + + // Network miscellanea + // HttpClient#send and sendAsync are abstract, so we instrument their internal implementations void check$jdk_internal_net_http_HttpClientImpl$send( Class callerClass, @@ -411,6 +637,16 @@ public interface EntitlementChecker { void check$sun_nio_ch_DatagramChannelImpl$receive(Class callerClass, DatagramChannel that, ByteBuffer dst); + // providers (SPI) + + // protected constructors + void check$java_nio_channels_spi_SelectorProvider$(Class callerClass); + + void check$java_nio_channels_spi_AsynchronousChannelProvider$(Class callerClass); + + // provider methods (dynamic) + void checkSelectorProviderInheritedChannel(Class callerClass, SelectorProvider that); + /// ///////////////// // // Load native libraries @@ -431,21 +667,555 @@ public interface EntitlementChecker { // File access // + // old io (ie File) + void check$java_io_File$canExecute(Class callerClass, File file); + + void check$java_io_File$canRead(Class callerClass, File file); + + void check$java_io_File$canWrite(Class callerClass, File file); + + void check$java_io_File$createNewFile(Class callerClass, File file); + + void check$java_io_File$$createTempFile(Class callerClass, String prefix, String suffix, File directory); + + void check$java_io_File$delete(Class callerClass, File file); + + void check$java_io_File$deleteOnExit(Class callerClass, File file); + + void check$java_io_File$exists(Class callerClass, File file); + + void check$java_io_File$isDirectory(Class callerClass, File file); + + void check$java_io_File$isFile(Class callerClass, File file); + + void check$java_io_File$isHidden(Class callerClass, File file); + + void check$java_io_File$lastModified(Class callerClass, File file); + + void check$java_io_File$length(Class callerClass, File file); + + void check$java_io_File$list(Class callerClass, File file); + + void check$java_io_File$list(Class callerClass, File file, FilenameFilter filter); + + void check$java_io_File$listFiles(Class callerClass, File file); + + void check$java_io_File$listFiles(Class callerClass, File file, FileFilter filter); + + void check$java_io_File$listFiles(Class callerClass, File file, FilenameFilter filter); + + void check$java_io_File$mkdir(Class callerClass, File file); + + void check$java_io_File$mkdirs(Class callerClass, File file); + + void check$java_io_File$renameTo(Class callerClass, File file, File dest); + + void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable); + + void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable, boolean ownerOnly); + + void check$java_io_File$setLastModified(Class callerClass, File file, long time); + + void check$java_io_File$setReadable(Class callerClass, File file, boolean readable); + + void check$java_io_File$setReadable(Class callerClass, File file, boolean readable, boolean ownerOnly); + + void check$java_io_File$setReadOnly(Class callerClass, File file); + + void check$java_io_File$setWritable(Class callerClass, File file, boolean writable); + + void check$java_io_File$setWritable(Class callerClass, File file, boolean writable, boolean ownerOnly); + + void check$java_io_FileInputStream$(Class callerClass, File file); + + void check$java_io_FileInputStream$(Class callerClass, FileDescriptor fd); + + void check$java_io_FileInputStream$(Class callerClass, String name); + + void check$java_io_FileOutputStream$(Class callerClass, File file); + + void check$java_io_FileOutputStream$(Class callerClass, File file, boolean append); + + void check$java_io_FileOutputStream$(Class callerClass, FileDescriptor fd); + + void check$java_io_FileOutputStream$(Class callerClass, String name); + + void check$java_io_FileOutputStream$(Class callerClass, String name, boolean append); + + void check$java_io_FileReader$(Class callerClass, File file); + + void check$java_io_FileReader$(Class callerClass, File file, Charset charset); + + void check$java_io_FileReader$(Class callerClass, FileDescriptor fd); + + void check$java_io_FileReader$(Class callerClass, String name); + + void check$java_io_FileReader$(Class callerClass, String name, Charset charset); + + void check$java_io_FileWriter$(Class callerClass, File file); + + void check$java_io_FileWriter$(Class callerClass, File file, boolean append); + + void check$java_io_FileWriter$(Class callerClass, File file, Charset charset); + + void check$java_io_FileWriter$(Class callerClass, File file, Charset charset, boolean append); + + void check$java_io_FileWriter$(Class callerClass, FileDescriptor fd); + + void check$java_io_FileWriter$(Class callerClass, String name); + + void check$java_io_FileWriter$(Class callerClass, String name, boolean append); + + void check$java_io_FileWriter$(Class callerClass, String name, Charset charset); + + void check$java_io_FileWriter$(Class callerClass, String name, Charset charset, boolean append); + + void check$java_io_RandomAccessFile$(Class callerClass, String name, String mode); + + void check$java_io_RandomAccessFile$(Class callerClass, File file, String mode); + + void check$java_security_KeyStore$$getInstance(Class callerClass, File file, char[] password); + + void check$java_security_KeyStore$$getInstance(Class callerClass, File file, KeyStore.LoadStoreParameter param); + + void check$java_security_KeyStore$Builder$$newInstance(Class callerClass, File file, KeyStore.ProtectionParameter protection); + + void check$java_security_KeyStore$Builder$$newInstance( + Class callerClass, + String type, + Provider provider, + File file, + KeyStore.ProtectionParameter protection + ); + void check$java_util_Scanner$(Class callerClass, File source); void check$java_util_Scanner$(Class callerClass, File source, String charsetName); void check$java_util_Scanner$(Class callerClass, File source, Charset charset); - void check$java_io_FileOutputStream$(Class callerClass, String name); + void check$java_util_jar_JarFile$(Class callerClass, String name); - void check$java_io_FileOutputStream$(Class callerClass, String name, boolean append); + void check$java_util_jar_JarFile$(Class callerClass, String name, boolean verify); - void check$java_io_FileOutputStream$(Class callerClass, File file); + void check$java_util_jar_JarFile$(Class callerClass, File file); - void check$java_io_FileOutputStream$(Class callerClass, File file, boolean append); + void check$java_util_jar_JarFile$(Class callerClass, File file, boolean verify); + + void check$java_util_jar_JarFile$(Class callerClass, File file, boolean verify, int mode); + + void check$java_util_jar_JarFile$(Class callerClass, File file, boolean verify, int mode, Runtime.Version version); + + void check$java_util_zip_ZipFile$(Class callerClass, String name); + + void check$java_util_zip_ZipFile$(Class callerClass, String name, Charset charset); + + void check$java_util_zip_ZipFile$(Class callerClass, File file); + + void check$java_util_zip_ZipFile$(Class callerClass, File file, int mode); + + void check$java_util_zip_ZipFile$(Class callerClass, File file, Charset charset); + + void check$java_util_zip_ZipFile$(Class callerClass, File file, int mode, Charset charset); + + // nio + // channels + void check$java_nio_channels_FileChannel$(Class callerClass); + + void check$java_nio_channels_FileChannel$$open( + Class callerClass, + Path path, + Set options, + FileAttribute... attrs + ); + + void check$java_nio_channels_FileChannel$$open(Class callerClass, Path path, OpenOption... options); + + void check$java_nio_channels_AsynchronousFileChannel$(Class callerClass); + + void check$java_nio_channels_AsynchronousFileChannel$$open( + Class callerClass, + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs + ); + + void check$java_nio_channels_AsynchronousFileChannel$$open(Class callerClass, Path path, OpenOption... options); + + void check$jdk_nio_Channels$$readWriteSelectableChannel( + Class callerClass, + FileDescriptor fd, + Channels.SelectableChannelCloser closer + ); + + // files + void check$java_nio_file_Files$$getOwner(Class callerClass, Path path, LinkOption... options); void check$java_nio_file_Files$$probeContentType(Class callerClass, Path path); void check$java_nio_file_Files$$setOwner(Class callerClass, Path path, UserPrincipal principal); + + void check$java_nio_file_Files$$newInputStream(Class callerClass, Path path, OpenOption... options); + + void check$java_nio_file_Files$$newOutputStream(Class callerClass, Path path, OpenOption... options); + + void check$java_nio_file_Files$$newByteChannel( + Class callerClass, + Path path, + Set options, + FileAttribute... attrs + ); + + void check$java_nio_file_Files$$newByteChannel(Class callerClass, Path path, OpenOption... options); + + void check$java_nio_file_Files$$newDirectoryStream(Class callerClass, Path dir); + + void check$java_nio_file_Files$$newDirectoryStream(Class callerClass, Path dir, String glob); + + void check$java_nio_file_Files$$newDirectoryStream(Class callerClass, Path dir, DirectoryStream.Filter filter); + + void check$java_nio_file_Files$$createFile(Class callerClass, Path path, FileAttribute... attrs); + + void check$java_nio_file_Files$$createDirectory(Class callerClass, Path dir, FileAttribute... attrs); + + void check$java_nio_file_Files$$createDirectories(Class callerClass, Path dir, FileAttribute... attrs); + + void check$java_nio_file_Files$$createTempFile(Class callerClass, Path dir, String prefix, String suffix, FileAttribute... attrs); + + void check$java_nio_file_Files$$createTempFile(Class callerClass, String prefix, String suffix, FileAttribute... attrs); + + void check$java_nio_file_Files$$createTempDirectory(Class callerClass, Path dir, String prefix, FileAttribute... attrs); + + void check$java_nio_file_Files$$createTempDirectory(Class callerClass, String prefix, FileAttribute... attrs); + + void check$java_nio_file_Files$$createSymbolicLink(Class callerClass, Path link, Path target, FileAttribute... attrs); + + void check$java_nio_file_Files$$createLink(Class callerClass, Path link, Path existing); + + void check$java_nio_file_Files$$delete(Class callerClass, Path path); + + void check$java_nio_file_Files$$deleteIfExists(Class callerClass, Path path); + + void check$java_nio_file_Files$$copy(Class callerClass, Path source, Path target, CopyOption... options); + + void check$java_nio_file_Files$$move(Class callerClass, Path source, Path target, CopyOption... options); + + void check$java_nio_file_Files$$readSymbolicLink(Class callerClass, Path link); + + void check$java_nio_file_Files$$getFileStore(Class callerClass, Path path); + + void check$java_nio_file_Files$$isSameFile(Class callerClass, Path path, Path path2); + + void check$java_nio_file_Files$$mismatch(Class callerClass, Path path, Path path2); + + void check$java_nio_file_Files$$isHidden(Class callerClass, Path path); + + void check$java_nio_file_Files$$getFileAttributeView( + Class callerClass, + Path path, + Class type, + LinkOption... options + ); + + void check$java_nio_file_Files$$readAttributes( + Class callerClass, + Path path, + Class type, + LinkOption... options + ); + + void check$java_nio_file_Files$$setAttribute(Class callerClass, Path path, String attribute, Object value, LinkOption... options); + + void check$java_nio_file_Files$$getAttribute(Class callerClass, Path path, String attribute, LinkOption... options); + + void check$java_nio_file_Files$$readAttributes(Class callerClass, Path path, String attributes, LinkOption... options); + + void check$java_nio_file_Files$$getPosixFilePermissions(Class callerClass, Path path, LinkOption... options); + + void check$java_nio_file_Files$$setPosixFilePermissions(Class callerClass, Path path, Set perms); + + void check$java_nio_file_Files$$isSymbolicLink(Class callerClass, Path path); + + void check$java_nio_file_Files$$isDirectory(Class callerClass, Path path, LinkOption... options); + + void check$java_nio_file_Files$$isRegularFile(Class callerClass, Path path, LinkOption... options); + + void check$java_nio_file_Files$$getLastModifiedTime(Class callerClass, Path path, LinkOption... options); + + void check$java_nio_file_Files$$setLastModifiedTime(Class callerClass, Path path, FileTime time); + + void check$java_nio_file_Files$$size(Class callerClass, Path path); + + void check$java_nio_file_Files$$exists(Class callerClass, Path path, LinkOption... options); + + void check$java_nio_file_Files$$notExists(Class callerClass, Path path, LinkOption... options); + + void check$java_nio_file_Files$$isReadable(Class callerClass, Path path); + + void check$java_nio_file_Files$$isWritable(Class callerClass, Path path); + + void check$java_nio_file_Files$$isExecutable(Class callerClass, Path path); + + void check$java_nio_file_Files$$walkFileTree( + Class callerClass, + Path start, + Set options, + int maxDepth, + FileVisitor visitor + ); + + void check$java_nio_file_Files$$walkFileTree(Class callerClass, Path start, FileVisitor visitor); + + void check$java_nio_file_Files$$newBufferedReader(Class callerClass, Path path, Charset cs); + + void check$java_nio_file_Files$$newBufferedReader(Class callerClass, Path path); + + void check$java_nio_file_Files$$newBufferedWriter(Class callerClass, Path path, Charset cs, OpenOption... options); + + void check$java_nio_file_Files$$newBufferedWriter(Class callerClass, Path path, OpenOption... options); + + void check$java_nio_file_Files$$copy(Class callerClass, InputStream in, Path target, CopyOption... options); + + void check$java_nio_file_Files$$copy(Class callerClass, Path source, OutputStream out); + + void check$java_nio_file_Files$$readAllBytes(Class callerClass, Path path); + + void check$java_nio_file_Files$$readString(Class callerClass, Path path); + + void check$java_nio_file_Files$$readString(Class callerClass, Path path, Charset cs); + + void check$java_nio_file_Files$$readAllLines(Class callerClass, Path path, Charset cs); + + void check$java_nio_file_Files$$readAllLines(Class callerClass, Path path); + + void check$java_nio_file_Files$$write(Class callerClass, Path path, byte[] bytes, OpenOption... options); + + void check$java_nio_file_Files$$write( + Class callerClass, + Path path, + Iterable lines, + Charset cs, + OpenOption... options + ); + + void check$java_nio_file_Files$$write(Class callerClass, Path path, Iterable lines, OpenOption... options); + + void check$java_nio_file_Files$$writeString(Class callerClass, Path path, CharSequence csq, OpenOption... options); + + void check$java_nio_file_Files$$writeString(Class callerClass, Path path, CharSequence csq, Charset cs, OpenOption... options); + + void check$java_nio_file_Files$$list(Class callerClass, Path dir); + + void check$java_nio_file_Files$$walk(Class callerClass, Path start, int maxDepth, FileVisitOption... options); + + void check$java_nio_file_Files$$walk(Class callerClass, Path start, FileVisitOption... options); + + void check$java_nio_file_Files$$find( + Class callerClass, + Path start, + int maxDepth, + BiPredicate matcher, + FileVisitOption... options + ); + + void check$java_nio_file_Files$$lines(Class callerClass, Path path, Charset cs); + + void check$java_nio_file_Files$$lines(Class callerClass, Path path); + + void check$java_nio_file_spi_FileSystemProvider$(Class callerClass); + + void check$java_util_logging_FileHandler$(Class callerClass); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, boolean append); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count, boolean append); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, long limit, int count, boolean append); + + void check$java_util_logging_FileHandler$close(Class callerClass, FileHandler that); + + void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class callerClass, Path path); + + void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path); + + void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path, OpenOption... options); + + void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload(Class callerClass, Path directory, OpenOption... openOptions); + + void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory); + + void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory, OpenOption... openOptions); + + void checkNewFileSystem(Class callerClass, FileSystemProvider that, URI uri, Map env); + + void checkNewFileSystem(Class callerClass, FileSystemProvider that, Path path, Map env); + + void checkNewInputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options); + + void checkNewOutputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options); + + void checkNewFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ); + + void checkNewAsynchronousFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs + ); + + void checkNewByteChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ); + + void checkNewDirectoryStream(Class callerClass, FileSystemProvider that, Path dir, DirectoryStream.Filter filter); + + void checkCreateDirectory(Class callerClass, FileSystemProvider that, Path dir, FileAttribute... attrs); + + void checkCreateSymbolicLink(Class callerClass, FileSystemProvider that, Path link, Path target, FileAttribute... attrs); + + void checkCreateLink(Class callerClass, FileSystemProvider that, Path link, Path existing); + + void checkDelete(Class callerClass, FileSystemProvider that, Path path); + + void checkDeleteIfExists(Class callerClass, FileSystemProvider that, Path path); + + void checkReadSymbolicLink(Class callerClass, FileSystemProvider that, Path link); + + void checkCopy(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options); + + void checkMove(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options); + + void checkIsSameFile(Class callerClass, FileSystemProvider that, Path path, Path path2); + + void checkIsHidden(Class callerClass, FileSystemProvider that, Path path); + + void checkGetFileStore(Class callerClass, FileSystemProvider that, Path path); + + void checkCheckAccess(Class callerClass, FileSystemProvider that, Path path, AccessMode... modes); + + void checkGetFileAttributeView(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options); + + void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options); + + void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, String attributes, LinkOption... options); + + void checkSetAttribute(Class callerClass, FileSystemProvider that, Path path, String attribute, Object value, LinkOption... options); + + // file store + void checkGetFileStoreAttributeView(Class callerClass, FileStore that, Class type); + + void checkGetAttribute(Class callerClass, FileStore that, String attribute); + + void checkGetBlockSize(Class callerClass, FileStore that); + + void checkGetTotalSpace(Class callerClass, FileStore that); + + void checkGetUnallocatedSpace(Class callerClass, FileStore that); + + void checkGetUsableSpace(Class callerClass, FileStore that); + + void checkIsReadOnly(Class callerClass, FileStore that); + + void checkName(Class callerClass, FileStore that); + + void checkType(Class callerClass, FileStore that); + + // path + void checkPathToRealPath(Class callerClass, Path that, LinkOption... options) throws NoSuchFileException; + + void checkPathRegister(Class callerClass, Path that, WatchService watcher, WatchEvent.Kind... events); + + void checkPathRegister( + Class callerClass, + Path that, + WatchService watcher, + WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers + ); + + // URLConnection + + void check$sun_net_www_protocol_file_FileURLConnection$connect(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_file_FileURLConnection$getHeaderFields(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_file_FileURLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, String name); + + void check$sun_net_www_protocol_file_FileURLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, int n); + + void check$sun_net_www_protocol_file_FileURLConnection$getContentLength(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_file_FileURLConnection$getContentLengthLong(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_file_FileURLConnection$getHeaderFieldKey(Class callerClass, java.net.URLConnection that, int n); + + void check$sun_net_www_protocol_file_FileURLConnection$getLastModified(Class callerClass, java.net.URLConnection that); + + void check$sun_net_www_protocol_file_FileURLConnection$getInputStream(Class callerClass, java.net.URLConnection that); + + void check$java_net_JarURLConnection$getManifest(Class callerClass, java.net.JarURLConnection that); + + void check$java_net_JarURLConnection$getJarEntry(Class callerClass, java.net.JarURLConnection that); + + void check$java_net_JarURLConnection$getAttributes(Class callerClass, java.net.JarURLConnection that); + + void check$java_net_JarURLConnection$getMainAttributes(Class callerClass, java.net.JarURLConnection that); + + void check$java_net_JarURLConnection$getCertificates(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getJarFile(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getJarEntry(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$connect(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getInputStream(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getContentLength(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getContentLengthLong(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getContent(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getContentType(Class callerClass, java.net.JarURLConnection that); + + void check$sun_net_www_protocol_jar_JarURLConnection$getHeaderField(Class callerClass, java.net.JarURLConnection that, String name); + + //////////////////// + // + // Thread management + // + + void check$java_lang_Thread$start(Class callerClass, Thread thread); + + void check$java_lang_Thread$setDaemon(Class callerClass, Thread thread, boolean on); + + void check$java_lang_ThreadGroup$setDaemon(Class callerClass, ThreadGroup threadGroup, boolean daemon); + + void check$java_util_concurrent_ForkJoinPool$setParallelism(Class callerClass, ForkJoinPool forkJoinPool, int size); + + void check$java_lang_Thread$setName(Class callerClass, Thread thread, String name); + + void check$java_lang_Thread$setPriority(Class callerClass, Thread thread, int newPriority); + + void check$java_lang_Thread$setUncaughtExceptionHandler(Class callerClass, Thread thread, Thread.UncaughtExceptionHandler ueh); + + void check$java_lang_ThreadGroup$setMaxPriority(Class callerClass, ThreadGroup threadGroup, int pri); } diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java index 26c9c83b8eb51..c0344e4a8c10c 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java @@ -18,6 +18,7 @@ public class EntitlementCheckerHandle { * This is how the bytecodes injected by our instrumentation access the {@link EntitlementChecker} * so they can call the appropriate check method. */ + @SuppressWarnings("unused") public static EntitlementChecker instance() { return Holder.instance; } diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/Util.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/Util.java new file mode 100644 index 0000000000000..9fd34d3b72c2c --- /dev/null +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/Util.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import java.util.Optional; + +import static java.lang.StackWalker.Option.RETAIN_CLASS_REFERENCE; + +public class Util { + /** + * A special value representing the case where a method has no caller. + * This can occur if it's called directly from the JVM. + * + * @see StackWalker#getCallerClass() + */ + public static final Class NO_CLASS = new Object() { + }.getClass(); + + /** + * Why would we write this instead of using {@link StackWalker#getCallerClass()}? + * Because that method throws {@link IllegalCallerException} if called from the "outermost frame", + * which includes at least some cases of a method called from a native frame. + * + * @return the class that called the method which called this; or {@link #NO_CLASS} from the outermost frame. + */ + @SuppressWarnings("unused") // Called reflectively from InstrumenterImpl + public static Class getCallerClass() { + Optional> callerClassIfAny = StackWalker.getInstance(RETAIN_CLASS_REFERENCE) + .walk( + frames -> frames.skip(2) // Skip this method and its caller + .findFirst() + .map(StackWalker.StackFrame::getDeclaringClass) + ); + return callerClassIfAny.orElse(NO_CLASS); + } + +} diff --git a/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19EntitlementChecker.java b/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19EntitlementChecker.java new file mode 100644 index 0000000000000..7f89d51bc7bfb --- /dev/null +++ b/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19EntitlementChecker.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +public interface Java19EntitlementChecker extends Java19PreviewEntitlementChecker, EntitlementChecker {} diff --git a/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19EntitlementCheckerHandle.java b/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19EntitlementCheckerHandle.java new file mode 100644 index 0000000000000..a13de64960412 --- /dev/null +++ b/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19EntitlementCheckerHandle.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +/** + * Java19 variant of {@link EntitlementChecker} handle holder. + */ +public class Java19EntitlementCheckerHandle { + + public static Java19EntitlementChecker instance() { + return Holder.instance; + } + + private static class Holder { + private static final Java19EntitlementChecker instance = HandleLoader.load(Java19EntitlementChecker.class); + } + + // no construction + private Java19EntitlementCheckerHandle() {} +} diff --git a/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19PreviewEntitlementChecker.java b/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19PreviewEntitlementChecker.java new file mode 100644 index 0000000000000..ae804a6a8f088 --- /dev/null +++ b/libs/entitlement/bridge/src/main19/java/org/elasticsearch/entitlement/bridge/Java19PreviewEntitlementChecker.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import java.lang.foreign.Addressable; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryAddress; +import java.lang.foreign.MemorySession; +import java.lang.invoke.MethodHandle; +import java.nio.file.Path; + +/** + * Interface with Java19 Preview specific functions and types. + * This interface must be kept isolated, as we cannot inherit from it in subsequent Java-specific versions as it contains types that + * were removed in the following previews/in final code (like MemorySession or MemoryAddress) + */ +public interface Java19PreviewEntitlementChecker { + + /** + * downcallHandle has a different signature in Java 19. + * See docs: https://docs.oracle.com/en/java/javase/19/docs/api/java.base/java/lang/foreign/Linker.html#downcallHandle(java.lang.foreign.FunctionDescriptor) + * + * Its only allowed implementation is in AbstractLinker: + * https://github.com/openjdk/jdk19u/blob/677bec11078ff41c21821fec46590752e0fc5128/src/java.base/share/classes/jdk/internal/foreign/abi/AbstractLinker.java#L47 + */ + void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle(Class callerClass, Linker that, FunctionDescriptor function); + + /** + * downcallHandle has a different signature in Java 19, and it is a default interface method. Later implementations (Java 21+) + * use an implementation class for this overload too. + * See docs: https://docs.oracle.com/en/java/javase/19/docs/api/java.base/java/lang/foreign/Linker.html#downcallHandle(java.lang.foreign.Addressable,java.lang.foreign.FunctionDescriptor) + */ + void check$java_lang_foreign_Linker$downcallHandle(Class callerClass, Linker that, Addressable address, FunctionDescriptor function); + + /** + * upcallStub has a different signature in Java 19, + * Its only allowed implementation is in AbstractLinker: + * https://github.com/openjdk/jdk19u/blob/677bec11078ff41c21821fec46590752e0fc5128/src/java.base/share/classes/jdk/internal/foreign/abi/AbstractLinker.java#L60 + */ + void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( + Class callerClass, + Linker that, + MethodHandle target, + FunctionDescriptor function, + MemorySession scope + ); + + /** + * This function has a different signature in Java 20. + * See docs: https://docs.oracle.com/en/java/javase/19/docs/api/java.base/java/lang/foreign/MemorySegment.html#ofAddress(java.lang.foreign.MemoryAddress,long,java.lang.foreign.MemorySession) + * + * It is superseded by {@code MemorySegment.reinterpret} in the final + * implementation (Java 21+) + */ + void check$java_lang_foreign_MemorySegment$$ofAddress( + Class callerClass, + MemoryAddress address, + long byteSize, + MemorySession session + ); + + /** + * This function signature changes from Java 19 to Java 20 (MemorySession parameter). + */ + void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, MemorySession session); + + /** + * This function signature changes from Java 19 to Java 20 (MemorySession parameter). + */ + void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, MemorySession session); +} diff --git a/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20EntitlementChecker.java b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20EntitlementChecker.java new file mode 100644 index 0000000000000..8b0172a42d670 --- /dev/null +++ b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20EntitlementChecker.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +public interface Java20EntitlementChecker extends Java20StableEntitlementChecker, Java20PreviewEntitlementChecker {} diff --git a/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20EntitlementCheckerHandle.java b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20EntitlementCheckerHandle.java new file mode 100644 index 0000000000000..b4283a1f66080 --- /dev/null +++ b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20EntitlementCheckerHandle.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +/** + * Java20 variant of {@link EntitlementChecker} handle holder. + */ +public class Java20EntitlementCheckerHandle { + + public static Java20EntitlementChecker instance() { + return Holder.instance; + } + + private static class Holder { + private static final Java20EntitlementChecker instance = HandleLoader.load(Java20EntitlementChecker.class); + } + + // no construction + private Java20EntitlementCheckerHandle() {} +} diff --git a/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20PreviewEntitlementChecker.java b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20PreviewEntitlementChecker.java new file mode 100644 index 0000000000000..012c628903204 --- /dev/null +++ b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20PreviewEntitlementChecker.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentScope; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.nio.file.Path; + +/** + * Interface with Java20 Preview specific functions and types. + * This interface must be kept isolated, as we cannot inherit from it in subsequent Java-specific versions as it contains types that + * were removed in the following previews/in final code (like MemorySession or MemoryAddress) + */ +public interface Java20PreviewEntitlementChecker { + + /** + * This downcallHandle overload has its final signature in Java 20. + * See docs: https://docs.oracle.com/en/java/javase/20/docs/api/java.base/java/lang/foreign/Linker.html#downcallHandle(java.lang.foreign.MemorySegment,java.lang.foreign.FunctionDescriptor,java.lang.foreign.Linker.Option...) + * + * However in Java 20 it is implemented as a default interface method. + * Later implementations (Java 21+) use an implementation class for this overload too, so we need a specific check method for this. + * See https://github.com/openjdk/jdk20u/blob/9ced461a4d8cb2ecfe2d6a74ec218ec589dcd617/src/java.base/share/classes/java/lang/foreign/Linker.java#L211 + */ + void check$java_lang_foreign_Linker$downcallHandle( + Class callerClass, + Linker that, + MemorySegment address, + FunctionDescriptor function, + Linker.Option... options + ); + + /** + * upcallStub has a different signature in Java 20 (SegmentScope parameter), + * Its only allowed implementation is in AbstractLinker: + * https://github.com/openjdk/jdk20u/blob/9ced461a4d8cb2ecfe2d6a74ec218ec589dcd617/src/java.base/share/classes/jdk/internal/foreign/abi/AbstractLinker.java#L69 + */ + void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( + Class callerClass, + Linker that, + MethodHandle target, + FunctionDescriptor function, + SegmentScope scope + ); + + /** + * This function signature changes from Java 19 to Java 20. + * It is superseded by {@code MemorySegment.reinterpret} in the final implementation (Java 21+) + */ + void check$java_lang_foreign_MemorySegment$$ofAddress(Class callerClass, long address); + + /** + * This function signature changes from Java 19 to Java 20. + * See docs: https://docs.oracle.com/en/java/javase/20/docs/api/java.base/java/lang/foreign/MemorySegment.html#ofAddress(long,long,java.lang.foreign.SegmentScope) + * + * It is superseded by {@code MemorySegment.reinterpret} in the final implementation (Java 21+) + * See https://github.com/openjdk/jdk20u/blob/9ced461a4d8cb2ecfe2d6a74ec218ec589dcd617/src/java.base/share/classes/java/lang/foreign/MemorySegment.java#L1071C5-L1071C64 + */ + void check$java_lang_foreign_MemorySegment$$ofAddress(Class callerClass, long address, long byteSize); + + /** + * This function overload is new to Java 20. + * It is superseded by {@code MemorySegment.reinterpret} in the final implementation (Java 21+) + */ + void check$java_lang_foreign_MemorySegment$$ofAddress(Class callerClass, long address, long byteSize, SegmentScope scope); + + /** + * This function overload is new to Java 20. + * It is superseded by {@code MemorySegment.reinterpret} in the final implementation (Java 21+) + */ + void check$java_lang_foreign_MemorySegment$$ofAddress( + Class callerClass, + long address, + long byteSize, + SegmentScope scope, + Runnable cleanupAction + ); + + /** + * This function is specific to Java 20. + * It is superseded by {@code MemorySegment.reinterpret} in the final implementation (Java 21+) + * See https://github.com/openjdk/jdk20u/blob/9ced461a4d8cb2ecfe2d6a74ec218ec589dcd617/src/java.base/share/classes/jdk/internal/foreign/layout/ValueLayouts.java#L442 + */ + void check$jdk_internal_foreign_layout_ValueLayouts$OfAddressImpl$asUnbounded(Class callerClass, ValueLayout.OfAddress that); + + /** + * This function signature changes from Java 20 to Java 21 (SegmentScope parameter). + */ + void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, SegmentScope scope); + + /** + * This function signature changes from Java 20 to Java 21 (SegmentScope parameter). + */ + void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, SegmentScope scope); +} diff --git a/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20StableEntitlementChecker.java b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20StableEntitlementChecker.java new file mode 100644 index 0000000000000..ad4d1ec2e2a43 --- /dev/null +++ b/libs/entitlement/bridge/src/main20/java/org/elasticsearch/entitlement/bridge/Java20StableEntitlementChecker.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.spi.FileSystemProvider; + +/** + * Interface with Java20 "stable" functions and types. + * It inherits from the previous "stable" interface, in this case {@link EntitlementChecker} as there is no Java19-specific stable + * API to instrument. + */ +public interface Java20StableEntitlementChecker extends EntitlementChecker { + + /** + * This overload of downcallHandle has its final form starting from Java 20 + * See docs: https://docs.oracle.com/en/java/javase/20/docs/api/java.base/java/lang/foreign/Linker.html#downcallHandle(java.lang.foreign.FunctionDescriptor,java.lang.foreign.Linker.Option...) + * + * Its only allowed implementation is in AbstractLinker: + * https://github.com/openjdk/jdk20u/blob/9ced461a4d8cb2ecfe2d6a74ec218ec589dcd617/src/java.base/share/classes/jdk/internal/foreign/abi/AbstractLinker.java#L52 + */ + void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( + Class callerClass, + Linker that, + FunctionDescriptor function, + Linker.Option... options + ); + + void checkReadAttributesIfExists(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options); + + void checkExists(Class callerClass, FileSystemProvider that, Path path, LinkOption... options); +} diff --git a/libs/entitlement/bridge/src/main21/java/org/elasticsearch/entitlement/bridge/Java21EntitlementChecker.java b/libs/entitlement/bridge/src/main21/java/org/elasticsearch/entitlement/bridge/Java21EntitlementChecker.java index e6add521d7229..372dfe1188bde 100644 --- a/libs/entitlement/bridge/src/main21/java/org/elasticsearch/entitlement/bridge/Java21EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main21/java/org/elasticsearch/entitlement/bridge/Java21EntitlementChecker.java @@ -9,4 +9,99 @@ package org.elasticsearch.entitlement.bridge; -public interface Java21EntitlementChecker extends EntitlementChecker {} +import java.lang.foreign.AddressLayout; +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.invoke.MethodHandle; +import java.nio.file.Path; +import java.util.function.Consumer; + +public interface Java21EntitlementChecker extends Java20StableEntitlementChecker { + + /** + * This function is in preview in Java 21, but it already has its final signature. + * See docs: https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/foreign/AddressLayout.html#withTargetLayout(java.lang.foreign.MemoryLayout) + * + * It has only one allowed implementation (interface is sealed). + * See https://github.com/openjdk/jdk21u/blob/7069f193f1f8c61869fc68a36c17f3a9a7b7b2a0/src/java.base/share/classes/jdk/internal/foreign/layout/ValueLayouts.java#L350 + */ + void check$jdk_internal_foreign_layout_ValueLayouts$OfAddressImpl$withTargetLayout( + Class callerClass, + AddressLayout that, + MemoryLayout memoryLayout + ); + + /** + * This function is in preview in Java 21, but it already has its final signature. + * See docs: https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/foreign/Linker.html#downcallHandle(java.lang.foreign.FunctionDescriptor,java.lang.foreign.Linker.Option...) + * + * It has only one allowed implementation (interface is sealed). + * See https://github.com/openjdk/jdk21u/blob/d2cbada0b7c88521dfb4d3696205c9beb77018af/src/java.base/share/classes/jdk/internal/foreign/abi/AbstractLinker.java#L77 + */ + void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( + Class callerClass, + Linker that, + MemorySegment address, + FunctionDescriptor function, + Linker.Option... options + ); + + /** + * This function is in preview in Java 21, but it already has its final signature. + * + * It has only one allowed implementation (interface is sealed). + * See https://github.com/openjdk/jdk21u/blob/d2cbada0b7c88521dfb4d3696205c9beb77018af/src/java.base/share/classes/jdk/internal/foreign/abi/AbstractLinker.java#L112 + */ + void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( + Class callerClass, + Linker that, + MethodHandle target, + FunctionDescriptor function, + Arena arena, + Linker.Option... options + ); + + /** + * This function is in preview in Java 21, but it already has its final signature. + * + * It has only one allowed implementation (interface is sealed). + * See https://github.com/openjdk/jdk21u/blob/d2cbada0b7c88521dfb4d3696205c9beb77018af/src/java.base/share/classes/jdk/internal/foreign/AbstractMemorySegmentImpl.java#L135 + */ + void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret(Class callerClass, MemorySegment that, long newSize); + + /** + * This function is in preview in Java 21, but it already has its final signature. + * It has only one allowed implementation in AbstractMemorySegmentImpl (interface is sealed). + */ + void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( + Class callerClass, + MemorySegment that, + long newSize, + Arena arena, + Consumer cleanup + ); + + /** + * This function is in preview in Java 21, but it already has its final signature. + * It has only one allowed implementation in AbstractMemorySegmentImpl (interface is sealed). + */ + void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( + Class callerClass, + MemorySegment that, + Arena arena, + Consumer cleanup + ); + + /** + * This function is in preview in Java 21, but it already has its final signature. + */ + void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, Arena arena); + + /** + * This function is in preview in Java 21, but it already has its final signature. + */ + void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, Arena arena); +} diff --git a/libs/entitlement/bridge/src/main22/java/org/elasticsearch/entitlement/bridge/Java22EntitlementChecker.java b/libs/entitlement/bridge/src/main22/java/org/elasticsearch/entitlement/bridge/Java22EntitlementChecker.java index 13e581c16a4fb..bdf9c84529188 100644 --- a/libs/entitlement/bridge/src/main22/java/org/elasticsearch/entitlement/bridge/Java22EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main22/java/org/elasticsearch/entitlement/bridge/Java22EntitlementChecker.java @@ -9,68 +9,4 @@ package org.elasticsearch.entitlement.bridge; -import java.lang.foreign.AddressLayout; -import java.lang.foreign.Arena; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.Linker; -import java.lang.foreign.MemoryLayout; -import java.lang.foreign.MemorySegment; -import java.lang.invoke.MethodHandle; -import java.nio.file.Path; -import java.util.function.Consumer; - -public interface Java22EntitlementChecker extends Java21EntitlementChecker { - // Sealed implementation of java.lang.foreign.AddressLayout - void check$jdk_internal_foreign_layout_ValueLayouts$OfAddressImpl$withTargetLayout( - Class callerClass, - AddressLayout that, - MemoryLayout memoryLayout - ); - - // Sealed implementation of java.lang.foreign.Linker - void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( - Class callerClass, - Linker that, - FunctionDescriptor function, - Linker.Option... options - ); - - void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( - Class callerClass, - Linker that, - MemorySegment address, - FunctionDescriptor function, - Linker.Option... options - ); - - void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( - Class callerClass, - Linker that, - MethodHandle target, - FunctionDescriptor function, - Arena arena, - Linker.Option... options - ); - - // Sealed implementation for java.lang.foreign.MemorySegment.reinterpret(long) - void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret(Class callerClass, MemorySegment that, long newSize); - - void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( - Class callerClass, - MemorySegment that, - long newSize, - Arena arena, - Consumer cleanup - ); - - void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( - Class callerClass, - MemorySegment that, - Arena arena, - Consumer cleanup - ); - - void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, Arena arena); - - void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, Arena arena); -} +public interface Java22EntitlementChecker extends Java21EntitlementChecker {} diff --git a/libs/entitlement/build.gradle b/libs/entitlement/build.gradle index b221851a5b020..0b8508323fbc7 100644 --- a/libs/entitlement/build.gradle +++ b/libs/entitlement/build.gradle @@ -28,6 +28,12 @@ dependencies { } // guarding for intellij + if (sourceSets.findByName("main19")) { + main19CompileOnly project(path: ':libs:entitlement:bridge', configuration: 'java19') + } + if (sourceSets.findByName("main20")) { + main20CompileOnly project(path: ':libs:entitlement:bridge', configuration: 'java20') + } if (sourceSets.findByName("main21")) { main21CompileOnly project(path: ':libs:entitlement:bridge', configuration: 'java21') } diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java b/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java index eafac9006daec..74559a12a4da4 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java @@ -12,6 +12,7 @@ requires org.elasticsearch.entitlement; requires org.elasticsearch.base; // SuppressForbidden requires org.elasticsearch.logging; + requires java.logging; exports org.elasticsearch.entitlement.qa.entitled; // Must be unqualified so non-modular IT tests can call us } diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java index 24d7472e07c65..289ce2dc2fe32 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledActions.java @@ -12,19 +12,118 @@ import org.elasticsearch.core.SuppressForbidden; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.UserPrincipal; +import java.security.SecureRandom; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +@SuppressForbidden(reason = "Exposes forbidden APIs for testing purposes") public final class EntitledActions { private EntitledActions() {} - @SuppressForbidden(reason = "Exposes forbidden APIs for testing purposes") - static void System_clearProperty(String key) { - System.clearProperty(key); + private static final SecureRandom random = new SecureRandom(); + + private static final Path testRootDir = Paths.get(System.getProperty("es.entitlements.testdir")); + + private static Path readDir() { + return testRootDir.resolve("read_dir"); + } + + private static Path readWriteDir() { + return testRootDir.resolve("read_write_dir"); } public static UserPrincipal getFileOwner(Path path) throws IOException { return Files.getOwner(path); } + + public static void createFile(Path path) throws IOException { + Files.createFile(path); + } + + public static Path createTempFileForRead() throws IOException { + return Files.createFile(readDir().resolve("entitlements-" + random.nextLong() + ".tmp")); + } + + public static Path createTempFileForWrite() throws IOException { + return Files.createFile(readWriteDir().resolve("entitlements-" + random.nextLong() + ".tmp")); + } + + public static Path createTempDirectoryForWrite() throws IOException { + return Files.createDirectory(readWriteDir().resolve("entitlements-dir-" + random.nextLong())); + } + + public static Path createTempSymbolicLink() throws IOException { + return createTempSymbolicLink(readWriteDir()); + } + + public static Path createTempSymbolicLink(Path target) throws IOException { + return Files.createSymbolicLink(readDir().resolve("entitlements-link-" + random.nextLong()), target); + } + + public static Path pathToRealPath(Path path) throws IOException { + return path.toRealPath(); + } + + public static Path createK8sLikeMount() throws IOException { + Path baseDir = readDir().resolve("k8s"); + var versionedDir = Files.createDirectories(baseDir.resolve("..version")); + var actualFileMount = Files.createFile(versionedDir.resolve("mount-" + random.nextLong() + ".tmp")); + + var dataDir = Files.createSymbolicLink(baseDir.resolve("..data"), versionedDir.getFileName()); + // mount-0.tmp -> ..data/mount-0.tmp -> ..version/mount-0.tmp + return Files.createSymbolicLink( + baseDir.resolve(actualFileMount.getFileName()), + dataDir.getFileName().resolve(actualFileMount.getFileName()) + ); + } + + public static URLConnection createHttpURLConnection() throws IOException { + return URI.create("http://127.0.0.1:12345/").toURL().openConnection(); + } + + public static URLConnection createHttpsURLConnection() throws IOException { + return URI.create("https://127.0.0.1:12345/").toURL().openConnection(); + } + + public static URLConnection createFtpURLConnection() throws IOException { + return URI.create("ftp://127.0.0.1:12345/").toURL().openConnection(); + } + + public static URLConnection createFileURLConnection() throws IOException { + var fileUrl = createTempFileForWrite().toUri().toURL(); + return fileUrl.openConnection(); + } + + public static URLConnection createMailToURLConnection() throws URISyntaxException, IOException { + return new URI("mailto", "email@example.com", null).toURL().openConnection(); + } + + public static Path createJar(Path dir, String name, Manifest manifest, String... files) throws IOException { + Path jarpath = dir.resolve(name); + try (var os = Files.newOutputStream(jarpath, StandardOpenOption.CREATE); var out = new JarOutputStream(os, manifest)) { + for (String file : files) { + out.putNextEntry(new JarEntry(file)); + } + } + return jarpath; + } + + public static URLConnection createJarURLConnection() throws IOException { + var manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + var tmpJarFile = createJar(readWriteDir(), "entitlements-" + random.nextLong() + ".jar", manifest, "a", "b"); + var jarFileUrl = tmpJarFile.toUri().toURL(); + var jarUrl = URI.create("jar:" + jarFileUrl + "!/a").toURL(); + return jarUrl.openConnection(); + } } diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java index 7a60d92ecc552..cec48ac168678 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/org/elasticsearch/entitlement/qa/entitled/EntitledPlugin.java @@ -15,7 +15,7 @@ import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; -import static org.elasticsearch.entitlement.qa.entitled.EntitledActions.System_clearProperty; +import java.util.concurrent.atomic.AtomicBoolean; public class EntitledPlugin extends Plugin implements ExtensiblePlugin { @@ -28,11 +28,19 @@ public static void selfTest() { selfTestNotEntitled(); } - private static final String SELF_TEST_PROPERTY = "org.elasticsearch.entitlement.qa.selfTest"; - private static void selfTestEntitled() { logger.debug("selfTestEntitled"); - System_clearProperty(SELF_TEST_PROPERTY); + AtomicBoolean threadRan = new AtomicBoolean(false); + try { + Thread testThread = new Thread(() -> threadRan.set(true), "testThread"); + testThread.start(); + testThread.join(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + if (threadRan.get() == false) { + throw new AssertionError("Self-test thread did not run"); + } } private static void selfTestNotEntitled() { diff --git a/libs/entitlement/qa/entitled-plugin/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitled-plugin/src/main/plugin-metadata/entitlement-policy.yaml deleted file mode 100644 index 81acd4c467f94..0000000000000 --- a/libs/entitlement/qa/entitled-plugin/src/main/plugin-metadata/entitlement-policy.yaml +++ /dev/null @@ -1,4 +0,0 @@ -org.elasticsearch.entitlement.qa.entitled: - - write_system_properties: - properties: - - org.elasticsearch.entitlement.qa.selfTest diff --git a/libs/entitlement/qa/entitlement-test-plugin/build.gradle b/libs/entitlement/qa/entitlement-test-plugin/build.gradle index d11eae32e1327..e19d78660aa93 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/build.gradle +++ b/libs/entitlement/qa/entitlement-test-plugin/build.gradle @@ -24,6 +24,7 @@ dependencies { compileOnly project(':server') compileOnly project(':libs:logging') compileOnly project(":libs:entitlement:qa:entitled-plugin") + implementation project(":libs:entitlement") } tasks.named("javadoc").configure { diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/module-info.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/module-info.java index bb4c6fd759426..aa8ab6aad4e17 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/module-info.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/module-info.java @@ -11,9 +11,12 @@ requires org.elasticsearch.server; requires org.elasticsearch.base; requires org.elasticsearch.logging; + requires org.elasticsearch.entitlement; requires org.elasticsearch.entitlement.qa.entitled; // Modules we'll attempt to use in order to exercise entitlements requires java.logging; requires java.net.http; + requires jdk.net; + requires java.desktop; } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java index 6564e0eed41e1..6529d63147272 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java @@ -9,7 +9,10 @@ package org.elasticsearch.entitlement.qa.test; -import java.io.IOException; +import jdk.nio.Channels; + +import org.elasticsearch.core.SuppressForbidden; + import java.io.InputStream; import java.io.OutputStream; import java.net.DatagramPacket; @@ -17,11 +20,47 @@ import java.net.DatagramSocketImpl; import java.net.InetAddress; import java.net.NetworkInterface; +import java.net.ProtocolFamily; import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress; import java.net.SocketException; import java.net.SocketImpl; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.AsynchronousChannelGroup; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.channels.AsynchronousServerSocketChannel; +import java.nio.channels.AsynchronousSocketChannel; +import java.nio.channels.CompletionHandler; +import java.nio.channels.DatagramChannel; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.Pipe; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.SelectableChannel; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.channels.spi.AbstractSelector; +import java.nio.channels.spi.AsynchronousChannelProvider; +import java.nio.channels.spi.SelectorProvider; +import java.nio.charset.Charset; +import java.nio.charset.spi.CharsetProvider; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; import java.security.cert.Certificate; import java.text.BreakIterator; import java.text.Collator; @@ -35,8 +74,13 @@ import java.text.spi.DateFormatSymbolsProvider; import java.text.spi.DecimalFormatSymbolsProvider; import java.text.spi.NumberFormatProvider; +import java.util.Iterator; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; import java.util.spi.CalendarDataProvider; import java.util.spi.CalendarNameProvider; import java.util.spi.CurrencyNameProvider; @@ -414,71 +458,71 @@ public Socket createSocket(Socket s, String host, int port, boolean autoClose) { } static class DummyDatagramSocket extends DatagramSocket { - DummyDatagramSocket() throws SocketException { + DummyDatagramSocket() { super(new DatagramSocketImpl() { @Override - protected void create() throws SocketException {} + protected void create() {} @Override - protected void bind(int lport, InetAddress laddr) throws SocketException {} + protected void bind(int lport, InetAddress laddr) {} @Override - protected void send(DatagramPacket p) throws IOException {} + protected void send(DatagramPacket p) {} @Override - protected int peek(InetAddress i) throws IOException { + protected int peek(InetAddress i) { return 0; } @Override - protected int peekData(DatagramPacket p) throws IOException { + protected int peekData(DatagramPacket p) { return 0; } @Override - protected void receive(DatagramPacket p) throws IOException {} + protected void receive(DatagramPacket p) {} @Override - protected void setTTL(byte ttl) throws IOException {} + protected void setTTL(byte ttl) {} @Override - protected byte getTTL() throws IOException { + protected byte getTTL() { return 0; } @Override - protected void setTimeToLive(int ttl) throws IOException {} + protected void setTimeToLive(int ttl) {} @Override - protected int getTimeToLive() throws IOException { + protected int getTimeToLive() { return 0; } @Override - protected void join(InetAddress inetaddr) throws IOException {} + protected void join(InetAddress inetaddr) {} @Override - protected void leave(InetAddress inetaddr) throws IOException {} + protected void leave(InetAddress inetaddr) {} @Override - protected void joinGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + protected void joinGroup(SocketAddress mcastaddr, NetworkInterface netIf) {} @Override - protected void leaveGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + protected void leaveGroup(SocketAddress mcastaddr, NetworkInterface netIf) {} @Override protected void close() {} @Override - public void setOption(int optID, Object value) throws SocketException {} + public void setOption(int optID, Object value) {} @Override - public Object getOption(int optID) throws SocketException { + public Object getOption(int optID) { return null; } @Override - protected void connect(InetAddress address, int port) throws SocketException {} + protected void connect(InetAddress address, int port) {} }); } } @@ -486,4 +530,320 @@ protected void connect(InetAddress address, int port) throws SocketException {} private static RuntimeException unexpected() { return new IllegalStateException("This method isn't supposed to be called"); } + + static class DummySelectorProvider extends SelectorProvider { + @Override + public DatagramChannel openDatagramChannel() { + return null; + } + + @Override + public DatagramChannel openDatagramChannel(ProtocolFamily family) { + return null; + } + + @Override + public Pipe openPipe() { + return null; + } + + @Override + public AbstractSelector openSelector() { + return null; + } + + @Override + public ServerSocketChannel openServerSocketChannel() { + return null; + } + + @Override + public SocketChannel openSocketChannel() { + return null; + } + } + + static class DummyAsynchronousChannelProvider extends AsynchronousChannelProvider { + @Override + public AsynchronousChannelGroup openAsynchronousChannelGroup(int nThreads, ThreadFactory threadFactory) { + return null; + } + + @Override + public AsynchronousChannelGroup openAsynchronousChannelGroup(ExecutorService executor, int initialSize) { + return null; + } + + @Override + public AsynchronousServerSocketChannel openAsynchronousServerSocketChannel(AsynchronousChannelGroup group) { + return null; + } + + @Override + public AsynchronousSocketChannel openAsynchronousSocketChannel(AsynchronousChannelGroup group) { + return null; + } + } + + static class DummyCharsetProvider extends CharsetProvider { + @Override + public Iterator charsets() { + return null; + } + + @Override + public Charset charsetForName(String charsetName) { + return null; + } + } + + static class DummyFileSystemProvider extends FileSystemProvider { + @Override + public String getScheme() { + return ""; + } + + @Override + public FileSystem newFileSystem(URI uri, Map env) { + return null; + } + + @Override + public FileSystem getFileSystem(URI uri) { + return null; + } + + @Override + public Path getPath(URI uri) { + return null; + } + + @Override + public SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) { + return null; + } + + @Override + public DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) { + return null; + } + + @Override + public void createDirectory(Path dir, FileAttribute... attrs) { + + } + + @Override + public void delete(Path path) { + + } + + @Override + public void copy(Path source, Path target, CopyOption... options) { + + } + + @Override + public void move(Path source, Path target, CopyOption... options) { + + } + + @Override + public boolean isSameFile(Path path, Path path2) { + return false; + } + + @Override + public boolean isHidden(Path path) { + return false; + } + + @Override + public FileStore getFileStore(Path path) { + return null; + } + + @Override + public void checkAccess(Path path, AccessMode... modes) { + + } + + @Override + public V getFileAttributeView(Path path, Class type, LinkOption... options) { + return null; + } + + @Override + public A readAttributes(Path path, Class type, LinkOption... options) { + return null; + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) { + return Map.of(); + } + + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) { + + } + } + + static class DummyFileChannel extends FileChannel { + @Override + protected void implCloseChannel() { + + } + + @Override + public int read(ByteBuffer dst) { + return 0; + } + + @Override + public long read(ByteBuffer[] dsts, int offset, int length) { + return 0; + } + + @Override + public int write(ByteBuffer src) { + return 0; + } + + @Override + public long write(ByteBuffer[] srcs, int offset, int length) { + return 0; + } + + @Override + public long position() { + return 0; + } + + @Override + public FileChannel position(long newPosition) { + return null; + } + + @Override + public long size() { + return 0; + } + + @Override + public FileChannel truncate(long size) { + return null; + } + + @Override + public void force(boolean metaData) { + + } + + @Override + public long transferTo(long position, long count, WritableByteChannel target) { + return 0; + } + + @Override + public long transferFrom(ReadableByteChannel src, long position, long count) { + return 0; + } + + @Override + public int read(ByteBuffer dst, long position) { + return 0; + } + + @Override + public int write(ByteBuffer src, long position) { + return 0; + } + + @Override + public MappedByteBuffer map(MapMode mode, long position, long size) { + return null; + } + + @Override + public FileLock lock(long position, long size, boolean shared) { + return null; + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) { + return null; + } + } + + static class DummyAsynchronousFileChannel extends AsynchronousFileChannel { + @Override + public boolean isOpen() { + return false; + } + + @Override + public void close() { + + } + + @Override + public long size() { + return 0; + } + + @Override + public AsynchronousFileChannel truncate(long size) { + return null; + } + + @Override + public void force(boolean metaData) { + + } + + @Override + public void lock(long position, long size, boolean shared, A attachment, CompletionHandler handler) { + + } + + @Override + public Future lock(long position, long size, boolean shared) { + return null; + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) { + return null; + } + + @Override + public void read(ByteBuffer dst, long position, A attachment, CompletionHandler handler) { + + } + + @Override + public Future read(ByteBuffer dst, long position) { + return null; + } + + @Override + public void write(ByteBuffer src, long position, A attachment, CompletionHandler handler) { + + } + + @Override + public Future write(ByteBuffer src, long position) { + return null; + } + } + + @SuppressForbidden(reason = "specifically testing readWriteSelectableChannel") + static class DummySelectableChannelCloser implements Channels.SelectableChannelCloser { + @Override + public void implCloseChannel(SelectableChannel sc) {} + + @Override + public void implReleaseChannel(SelectableChannel sc) {} + } } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTest.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTest.java new file mode 100644 index 0000000000000..840a031b4b672 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTest.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface EntitlementTest { + enum ExpectedAccess { + PLUGINS, + ES_MODULES_ONLY, + SERVER_ONLY, + ALWAYS_DENIED, + ALWAYS_ALLOWED + } + + ExpectedAccess expectedAccess(); + + Class expectedExceptionIfDenied() default NotEntitledException.class; + + int fromJavaVersion() default -1; +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTestPlugin.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTestPlugin.java index 36283cce3c81d..788c5738b6d63 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTestPlugin.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTestPlugin.java @@ -15,17 +15,28 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.env.Environment; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import java.util.Collection; import java.util.List; import java.util.function.Predicate; import java.util.function.Supplier; public class EntitlementTestPlugin extends Plugin implements ActionPlugin { + + private Environment environment; + + @Override + public Collection createComponents(PluginServices services) { + environment = services.environment(); + return super.createComponents(services); + } + @Override public List getRestHandlers( final Settings settings, @@ -38,6 +49,6 @@ public List getRestHandlers( final Supplier nodesInCluster, Predicate clusterSupportsFeature ) { - return List.of(new RestEntitlementsCheckAction()); + return List.of(new RestEntitlementsCheckAction(environment)); } } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java index 6e15ff4d0cdd1..e80b0a8580b5e 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java @@ -9,75 +9,594 @@ package org.elasticsearch.entitlement.qa.test; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.qa.entitled.EntitledActions; +import org.elasticsearch.env.Environment; +import java.io.File; +import java.io.FileDescriptor; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URISyntaxException; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.UserPrincipal; +import java.security.GeneralSecurityException; +import java.security.KeyStore; import java.util.Scanner; +import java.util.jar.JarFile; +import java.util.logging.FileHandler; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + +import javax.imageio.stream.FileImageInputStream; + +import static java.nio.charset.Charset.defaultCharset; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.WRITE; +import static java.util.zip.ZipFile.OPEN_DELETE; +import static java.util.zip.ZipFile.OPEN_READ; +import static org.elasticsearch.entitlement.qa.entitled.EntitledActions.createTempFileForWrite; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_ALLOWED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; @SuppressForbidden(reason = "Explicitly checking APIs that are forbidden") +@SuppressWarnings({ "unused" /* called via reflection */, "ResultOfMethodCallIgnored" }) class FileCheckActions { - private static Path testRootDir = Paths.get(System.getProperty("es.entitlements.testdir")); + static Path testRootDir = Paths.get(System.getProperty("es.entitlements.testdir")); - private static Path readDir() { + static Path readDir() { return testRootDir.resolve("read_dir"); } - private static Path readWriteDir() { + static Path readWriteDir() { return testRootDir.resolve("read_write_dir"); } - private static Path readFile() { + static Path readFile() { return testRootDir.resolve("read_file"); } - private static Path readWriteFile() { + static Path readWriteFile() { return testRootDir.resolve("read_write_file"); } - static void createScannerFile() throws FileNotFoundException { - new Scanner(readFile().toFile()); + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCanExecute() { + readFile().toFile().canExecute(); } - static void createScannerFileWithCharset() throws IOException { - new Scanner(readFile().toFile(), StandardCharsets.UTF_8); + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCanRead() { + readFile().toFile().canRead(); } - static void createScannerFileWithCharsetName() throws FileNotFoundException { - new Scanner(readFile().toFile(), "UTF-8"); + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCanWrite() { + readFile().toFile().canWrite(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCreateNewFile() throws IOException { + readWriteDir().resolve("new_file").toFile().createNewFile(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileCreateTempFile() throws IOException { + File.createTempFile("prefix", "suffix", readWriteDir().toFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileDelete() throws IOException { + var toDelete = EntitledActions.createTempFileForWrite(); + toDelete.toFile().delete(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileDeleteOnExit() throws IOException { + var toDelete = EntitledActions.createTempFileForWrite(); + toDelete.toFile().deleteOnExit(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileExists() { + readFile().toFile().exists(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileIsDirectory() { + readFile().toFile().isDirectory(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileIsFile() { + readFile().toFile().isFile(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileIsHidden() { + readFile().toFile().isHidden(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileLastModified() { + readFile().toFile().lastModified(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileLength() { + readFile().toFile().length(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileList() { + readDir().toFile().list(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileListWithFilter() { + readDir().toFile().list((dir, name) -> true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileListFiles() { + readDir().toFile().listFiles(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileListFilesWithFileFilter() { + readDir().toFile().listFiles(pathname -> true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileListFilesWithFilenameFilter() { + readDir().toFile().listFiles((dir, name) -> true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileMkdir() { + Path mkdir = readWriteDir().resolve("mkdir"); + mkdir.toFile().mkdir(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileMkdirs() { + Path mkdir = readWriteDir().resolve("mkdirs"); + mkdir.toFile().mkdirs(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileRenameTo() throws IOException { + var dir = EntitledActions.createTempDirectoryForWrite(); + Path toRename = dir.resolve("to_rename"); + EntitledActions.createFile(toRename); + toRename.toFile().renameTo(dir.resolve("renamed").toFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetExecutable() { + readWriteFile().toFile().setExecutable(false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetExecutableOwner() { + readWriteFile().toFile().setExecutable(false, false); } + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetLastModified() { + readWriteFile().toFile().setLastModified(System.currentTimeMillis()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetReadable() { + readWriteFile().toFile().setReadable(true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetReadableOwner() { + readWriteFile().toFile().setReadable(true, false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetReadOnly() throws IOException { + Path readOnly = EntitledActions.createTempFileForWrite(); + readOnly.toFile().setReadOnly(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetWritable() { + readWriteFile().toFile().setWritable(true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileSetWritableOwner() { + readWriteFile().toFile().setWritable(true, false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileInputStreamFile() throws IOException { + new FileInputStream(readFile().toFile()).close(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createFileInputStreamFileDescriptor() throws IOException { + new FileInputStream(FileDescriptor.in).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileInputStreamString() throws IOException { + new FileInputStream(readFile().toString()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) static void createFileOutputStreamString() throws IOException { new FileOutputStream(readWriteFile().toString()).close(); } + @EntitlementTest(expectedAccess = PLUGINS) static void createFileOutputStreamStringWithAppend() throws IOException { new FileOutputStream(readWriteFile().toString(), false).close(); } + @EntitlementTest(expectedAccess = PLUGINS) static void createFileOutputStreamFile() throws IOException { new FileOutputStream(readWriteFile().toFile()).close(); } + @EntitlementTest(expectedAccess = PLUGINS) static void createFileOutputStreamFileWithAppend() throws IOException { new FileOutputStream(readWriteFile().toFile(), false).close(); } - static void filesProbeContentType() throws IOException { - Files.probeContentType(readFile()); + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createFileOutputStreamFileDescriptor() throws IOException { + new FileOutputStream(FileDescriptor.out).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileReaderFile() throws IOException { + new FileReader(readFile().toFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileReaderFileCharset() throws IOException { + new FileReader(readFile().toFile(), StandardCharsets.UTF_8).close(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createFileReaderFileDescriptor() throws IOException { + new FileReader(FileDescriptor.in).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileReaderString() throws IOException { + new FileReader(readFile().toString()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileReaderStringCharset() throws IOException { + new FileReader(readFile().toString(), StandardCharsets.UTF_8).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterFile() throws IOException { + new FileWriter(readWriteFile().toFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterFileWithAppend() throws IOException { + new FileWriter(readWriteFile().toFile(), false).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterFileCharsetWithAppend() throws IOException { + new FileWriter(readWriteFile().toFile(), StandardCharsets.UTF_8, false).close(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createFileWriterFileDescriptor() throws IOException { + new FileWriter(FileDescriptor.out).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterString() throws IOException { + new FileWriter(readWriteFile().toString()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterStringWithAppend() throws IOException { + new FileWriter(readWriteFile().toString(), false).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterStringCharset() throws IOException { + new FileWriter(readWriteFile().toString(), StandardCharsets.UTF_8).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createFileWriterStringCharsetWithAppend() throws IOException { + new FileWriter(readWriteFile().toString(), StandardCharsets.UTF_8, false).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createRandomAccessFileStringRead() throws IOException { + new RandomAccessFile(readFile().toString(), "r").close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createRandomAccessFileStringReadWrite() throws IOException { + new RandomAccessFile(readWriteFile().toString(), "rw").close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createRandomAccessFileRead() throws IOException { + new RandomAccessFile(readFile().toFile(), "r").close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createRandomAccessFileReadWrite() throws IOException { + new RandomAccessFile(readWriteFile().toFile(), "rw").close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void keystoreGetInstance_FileCharArray() throws IOException { + try { + KeyStore.getInstance(readFile().toFile(), new char[0]); + } catch (GeneralSecurityException expected) { + return; + } + throw new AssertionError("Expected an exception"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void keystoreGetInstance_FileLoadStoreParameter() throws IOException { + try { + KeyStore.LoadStoreParameter loadStoreParameter = () -> null; + KeyStore.getInstance(readFile().toFile(), loadStoreParameter); + } catch (GeneralSecurityException expected) { + return; + } + throw new AssertionError("Expected an exception"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + @SuppressWarnings("DataFlowIssue") // Passing null to a @NotNull parameter + static void keystoreBuilderNewInstance() { + try { + KeyStore.Builder.newInstance("", null, readFile().toFile(), null); + } catch (NullPointerException expected) { + return; + } + throw new AssertionError("Expected an exception"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_String() throws IOException { + expectZipException(() -> new ZipFile(readFile().toString()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_StringCharset() throws IOException { + expectZipException(() -> new ZipFile(readFile().toString(), defaultCharset()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_File() throws IOException { + expectZipException(() -> new ZipFile(readFile().toFile()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_FileCharset() throws IOException { + expectZipException(() -> new ZipFile(readFile().toFile(), defaultCharset()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_FileReadOnly() throws IOException { + expectZipException(() -> new ZipFile(readFile().toFile(), OPEN_READ).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_FileReadAndDelete() throws IOException { + expectZipException(() -> new ZipFile(createTempFileForWrite().toFile(), OPEN_READ | OPEN_DELETE).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_ReadOnlyCharset() throws IOException { + expectZipException(() -> new ZipFile(readFile().toFile(), OPEN_READ, defaultCharset()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void zipFile_ReadAndDeleteCharset() throws IOException { + expectZipException(() -> new ZipFile(createTempFileForWrite().toFile(), OPEN_READ | OPEN_DELETE, defaultCharset()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_String() throws IOException { + expectZipException(() -> new JarFile(readFile().toString()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_StringBoolean() throws IOException { + expectZipException(() -> new JarFile(readFile().toString(), false).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_FileReadOnly() throws IOException { + expectZipException(() -> new JarFile(readFile().toFile(), false, OPEN_READ).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_FileReadAndDelete() throws IOException { + expectZipException(() -> new JarFile(createTempFileForWrite().toFile(), false, OPEN_READ | OPEN_DELETE).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_FileBooleanReadOnlyVersion() throws IOException { + expectZipException(() -> new JarFile(readFile().toFile(), false, OPEN_READ, Runtime.version()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_FileBooleanReadAndDeleteOnlyVersion() throws IOException { + expectZipException(() -> new JarFile(createTempFileForWrite().toFile(), false, OPEN_READ | OPEN_DELETE, Runtime.version()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFile_File() throws IOException { + expectZipException(() -> new JarFile(readFile().toFile()).close()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void jarFileFileBoolean() throws IOException { + expectZipException(() -> new JarFile(readFile().toFile(), false).close()); + } + + private static void expectZipException(CheckedRunnable action) throws IOException { + try { + action.run(); + } catch (ZipException expected) { + return; + } + throw new AssertionError("Expected an exception"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createScannerFile() throws FileNotFoundException { + new Scanner(readFile().toFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createScannerFileWithCharset() throws IOException { + new Scanner(readFile().toFile(), StandardCharsets.UTF_8); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createScannerFileWithCharsetName() throws FileNotFoundException { + new Scanner(readFile().toFile(), "UTF-8"); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler() throws IOException { + new FileHandler(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_String() throws IOException { + new FileHandler(readFile().toString()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringBoolean() throws IOException { + new FileHandler(readFile().toString(), false); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringIntInt() throws IOException { + new FileHandler(readFile().toString(), 1, 2); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringIntIntBoolean() throws IOException { + new FileHandler(readFile().toString(), 1, 2, false); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringLongIntBoolean() throws IOException { + new FileHandler(readFile().toString(), 1L, 2, false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpRequestBodyPublishersOfFile() throws IOException { + HttpRequest.BodyPublishers.ofFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodyHandlersOfFile() { + HttpResponse.BodyHandlers.ofFile(readWriteFile()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodyHandlersOfFile_readOnly() { + HttpResponse.BodyHandlers.ofFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodyHandlersOfFileDownload() { + HttpResponse.BodyHandlers.ofFileDownload(readWriteDir()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodyHandlersOfFileDownload_readOnly() { + HttpResponse.BodyHandlers.ofFileDownload(readDir()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodySubscribersOfFile_File() { + HttpResponse.BodySubscribers.ofFile(readWriteFile()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodySubscribersOfFile_File_readOnly() { + HttpResponse.BodySubscribers.ofFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodySubscribersOfFile_FileOpenOptions() { + // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed + // an overload distinct from ofFile with no OpenOptions, and so it needs its + // own instrumentation and its own test. + HttpResponse.BodySubscribers.ofFile(readWriteFile(), CREATE, WRITE); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodySubscribersOfFile_FileOpenOptions_readOnly() { + // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed + // an overload distinct from ofFile with no OpenOptions, and so it needs its + // own instrumentation and its own test. + HttpResponse.BodySubscribers.ofFile(readFile(), CREATE, WRITE); + } + + @EntitlementTest(expectedAccess = ALWAYS_ALLOWED) + static void readAccessConfigDirectory(Environment environment) { + Files.exists(environment.configDir()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void writeAccessConfigDirectory(Environment environment) throws IOException { + var file = environment.configDir().resolve("to_create"); + Files.createFile(file); + } + + @EntitlementTest(expectedAccess = ALWAYS_ALLOWED) + static void readAccessSourcePath() throws URISyntaxException { + var sourcePath = Paths.get(EntitlementTestPlugin.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + Files.exists(sourcePath); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void writeAccessSourcePath() throws IOException, URISyntaxException { + var sourcePath = Paths.get(EntitlementTestPlugin.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + var file = sourcePath.getParent().resolve("to_create"); + Files.createFile(file); } - static void filesSetOwner() throws IOException { - UserPrincipal owner = EntitledActions.getFileOwner(readWriteFile()); - Files.setOwner(readWriteFile(), owner); // set to existing owner, just trying to execute the method + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void javaDesktopFileAccess() throws Exception { + // Test file access from a java.desktop class. We explicitly exclude that module from the "system modules", so we expect + // any sensitive operation from java.desktop to fail. + var file = EntitledActions.createTempFileForRead(); + new FileImageInputStream(file.toFile()).close(); } private FileCheckActions() {} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java new file mode 100644 index 0000000000000..3482f267c53e8 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileStoreActions.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.FileStoreAttributeView; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class FileStoreActions { + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkGetFileStoreAttributeView() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getFileStoreAttributeView(FileStoreAttributeView.class); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetAttribute() throws IOException { + try { + Files.getFileStore(FileCheckActions.readFile()).getAttribute("zfs:compression"); + } catch (UnsupportedOperationException e) { + // It's OK if the attribute view is not available or it does not support reading the attribute + } + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetBlockSize() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getBlockSize(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetTotalSpace() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getTotalSpace(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetUnallocatedSpace() throws IOException { + Files.getFileStore(FileCheckActions.readWriteFile()).getUnallocatedSpace(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkGetUsableSpace() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).getUsableSpace(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkIsReadOnly() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).isReadOnly(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkName() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).name(); + } + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void checkType() throws IOException { + Files.getFileStore(FileCheckActions.readFile()).type(); + } + + private FileStoreActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/WritePropertiesCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/JvmActions.java similarity index 55% rename from libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/WritePropertiesCheckActions.java rename to libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/JvmActions.java index ba4bfa5e896b9..29e4ffccce0b3 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/WritePropertiesCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/JvmActions.java @@ -12,13 +12,20 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.qa.entitled.EntitledPlugin; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; import java.util.Locale; import java.util.TimeZone; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + @SuppressForbidden(reason = "testing entitlements") -class WritePropertiesCheckActions { - private WritePropertiesCheckActions() {} +@SuppressWarnings({ "unused" /* called via reflection */ }) +class JvmActions { + @EntitlementTest(expectedAccess = PLUGINS) static void setSystemProperty() { System.setProperty("es.entitlements.checkSetSystemProperty", "true"); try { @@ -29,24 +36,49 @@ static void setSystemProperty() { } + @EntitlementTest(expectedAccess = PLUGINS) static void clearSystemProperty() { EntitledPlugin.selfTest(); // TODO: find a better home System.clearProperty("es.entitlements.checkClearSystemProperty"); } + @EntitlementTest(expectedAccess = ALWAYS_DENIED) static void setSystemProperties() { System.setProperties(System.getProperties()); // no side effect in case if allowed (but shouldn't) } + @EntitlementTest(expectedAccess = ALWAYS_DENIED) static void setDefaultLocale() { Locale.setDefault(Locale.getDefault()); } + @EntitlementTest(expectedAccess = ALWAYS_DENIED) static void setDefaultLocaleForCategory() { Locale.setDefault(Locale.Category.DISPLAY, Locale.getDefault(Locale.Category.DISPLAY)); } + @EntitlementTest(expectedAccess = ALWAYS_DENIED) static void setDefaultTimeZone() { TimeZone.setDefault(TimeZone.getDefault()); } + + @EntitlementTest(expectedAccess = PLUGINS) + static void createClassLoader() throws IOException { + try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) { + // intentionally empty, just let the loader close + } + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createLogManager() { + new java.util.logging.LogManager() { + }; + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void thread$$setDefaultUncaughtExceptionHandler() { + Thread.setDefaultUncaughtExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()); + } + + private JvmActions() {} } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/LoadNativeLibrariesCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/LoadNativeLibrariesCheckActions.java index 50980bc230f55..7d2cc98f76f28 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/LoadNativeLibrariesCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/LoadNativeLibrariesCheckActions.java @@ -9,23 +9,30 @@ package org.elasticsearch.entitlement.qa.test; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressWarnings({ "unused" /* called via reflection */ }) class LoadNativeLibrariesCheckActions { + + @EntitlementTest(expectedAccess = PLUGINS) static void runtimeLoad() { try { - Runtime.getRuntime().load("libSomeLibFile.so"); + Runtime.getRuntime().load(FileCheckActions.readDir().resolve("libSomeLibFile.so").toString()); } catch (UnsatisfiedLinkError ignored) { // The library does not exist, so we expect to fail loading it } } + @EntitlementTest(expectedAccess = PLUGINS) static void systemLoad() { try { - System.load("libSomeLibFile.so"); + System.load(FileCheckActions.readDir().resolve("libSomeLibFile.so").toString()); } catch (UnsatisfiedLinkError ignored) { // The library does not exist, so we expect to fail loading it } } + @EntitlementTest(expectedAccess = PLUGINS) static void runtimeLoadLibrary() { try { Runtime.getRuntime().loadLibrary("SomeLib"); @@ -34,6 +41,7 @@ static void runtimeLoadLibrary() { } } + @EntitlementTest(expectedAccess = PLUGINS) static void systemLoadLibrary() { try { System.loadLibrary("SomeLib"); @@ -41,4 +49,6 @@ static void systemLoadLibrary() { // The library does not exist, so we expect to fail loading it } } + + private LoadNativeLibrariesCheckActions() {} } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java new file mode 100644 index 0000000000000..e2a4a080dc5cc --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/ManageThreadsActions.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.SuppressForbidden; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.lang.Thread.currentThread; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressForbidden(reason = "testing entitlements") +@SuppressWarnings({ "unused" /* called via reflaction */, "removal" }) +class ManageThreadsActions { + private ManageThreadsActions() {} + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$start() throws InterruptedException { + AtomicBoolean threadRan = new AtomicBoolean(false); + Thread thread = new Thread(() -> threadRan.set(true), "test"); + thread.start(); + thread.join(); + assert threadRan.get(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setDaemon() { + new Thread().setDaemon(true); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_ThreadGroup$setDaemon() { + currentThread().getThreadGroup().setDaemon(currentThread().getThreadGroup().isDaemon()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setName() { + currentThread().setName(currentThread().getName()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setPriority() { + currentThread().setPriority(currentThread().getPriority()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_Thread$setUncaughtExceptionHandler() { + currentThread().setUncaughtExceptionHandler(currentThread().getUncaughtExceptionHandler()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_lang_ThreadGroup$setMaxPriority() { + currentThread().getThreadGroup().setMaxPriority(currentThread().getThreadGroup().getMaxPriority()); + } + +} diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NativeActions.java similarity index 85% rename from qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy rename to libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NativeActions.java index f554bc62d7311..3493b64b31057 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NativeActions.java @@ -7,6 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -grant { - permission java.util.PropertyPermission "someProperty", "read"; -}; +package org.elasticsearch.entitlement.qa.test; + +class NativeActions {} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NetworkAccessCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NetworkAccessCheckActions.java index f0929894c262c..c92551488a773 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NetworkAccessCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NetworkAccessCheckActions.java @@ -12,14 +12,23 @@ import org.elasticsearch.core.SuppressForbidden; import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.NetworkInterface; import java.net.Proxy; +import java.net.ProxySelector; +import java.net.ResponseCache; import java.net.ServerSocket; import java.net.Socket; import java.net.SocketException; -import java.net.URI; -import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.net.spi.URLStreamHandlerProvider; import java.nio.ByteBuffer; import java.nio.channels.AsynchronousServerSocketChannel; import java.nio.channels.AsynchronousSocketChannel; @@ -34,9 +43,17 @@ import java.util.Arrays; import java.util.concurrent.ExecutionException; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + @SuppressForbidden(reason = "Testing entitlement check on forbidden action") +@SuppressWarnings({ "unused" /* called via reflection */, "deprecation" }) class NetworkAccessCheckActions { + @EntitlementTest(expectedAccess = PLUGINS) static void serverSocketAccept() throws IOException { try (ServerSocket socket = new DummyImplementations.DummyBoundServerSocket()) { try { @@ -51,37 +68,36 @@ static void serverSocketAccept() throws IOException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void serverSocketBind() throws IOException { try (ServerSocket socket = new DummyImplementations.DummyServerSocket()) { socket.bind(null); } } + @EntitlementTest(expectedAccess = PLUGINS) static void createSocketWithProxy() throws IOException { try (Socket socket = new Socket(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(0)))) { assert socket.isBound() == false; } } + @EntitlementTest(expectedAccess = PLUGINS) static void socketBind() throws IOException { try (Socket socket = new DummyImplementations.DummySocket()) { socket.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } + @EntitlementTest(expectedAccess = PLUGINS) static void socketConnect() throws IOException { try (Socket socket = new DummyImplementations.DummySocket()) { socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } - static void urlOpenConnectionWithProxy() throws URISyntaxException, IOException { - var url = new URI("http://localhost").toURL(); - var urlConnection = url.openConnection(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(0))); - assert urlConnection != null; - } - - static void createLDAPCertStore() throws NoSuchAlgorithmException { + @EntitlementTest(expectedAccess = PLUGINS) + static void createLDAPCertStore() { try { // We pass down null params to provoke a InvalidAlgorithmParameterException CertStore.getInstance("LDAP", null); @@ -94,18 +110,21 @@ static void createLDAPCertStore() throws NoSuchAlgorithmException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void serverSocketChannelBind() throws IOException { try (var serverSocketChannel = ServerSocketChannel.open()) { serverSocketChannel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } + @EntitlementTest(expectedAccess = PLUGINS) static void serverSocketChannelBindWithBacklog() throws IOException { try (var serverSocketChannel = ServerSocketChannel.open()) { serverSocketChannel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 50); } } + @EntitlementTest(expectedAccess = PLUGINS) static void serverSocketChannelAccept() throws IOException { try (var serverSocketChannel = ServerSocketChannel.open()) { serverSocketChannel.configureBlocking(false); @@ -118,18 +137,21 @@ static void serverSocketChannelAccept() throws IOException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousServerSocketChannelBind() throws IOException { try (var serverSocketChannel = AsynchronousServerSocketChannel.open()) { serverSocketChannel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousServerSocketChannelBindWithBacklog() throws IOException { try (var serverSocketChannel = AsynchronousServerSocketChannel.open()) { serverSocketChannel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 50); } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousServerSocketChannelAccept() throws IOException { try (var serverSocketChannel = AsynchronousServerSocketChannel.open()) { try { @@ -142,6 +164,7 @@ static void asynchronousServerSocketChannelAccept() throws IOException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousServerSocketChannelAcceptWithHandler() throws IOException { try (var serverSocketChannel = AsynchronousServerSocketChannel.open()) { try { @@ -161,12 +184,14 @@ public void failed(Throwable exc, Object attachment) { } } + @EntitlementTest(expectedAccess = PLUGINS) static void socketChannelBind() throws IOException { try (var socketChannel = SocketChannel.open()) { socketChannel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } + @EntitlementTest(expectedAccess = PLUGINS) static void socketChannelConnect() throws IOException { try (var socketChannel = SocketChannel.open()) { try { @@ -178,12 +203,14 @@ static void socketChannelConnect() throws IOException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousSocketChannelBind() throws IOException { try (var socketChannel = AsynchronousSocketChannel.open()) { socketChannel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousSocketChannelConnect() throws IOException, InterruptedException { try (var socketChannel = AsynchronousSocketChannel.open()) { var future = socketChannel.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); @@ -197,6 +224,7 @@ static void asynchronousSocketChannelConnect() throws IOException, InterruptedEx } } + @EntitlementTest(expectedAccess = PLUGINS) static void asynchronousSocketChannelConnectWithCompletion() throws IOException { try (var socketChannel = AsynchronousSocketChannel.open()) { socketChannel.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), null, new CompletionHandler<>() { @@ -211,12 +239,14 @@ public void failed(Throwable exc, Object attachment) { } } + @EntitlementTest(expectedAccess = PLUGINS) static void datagramChannelBind() throws IOException { try (var channel = DatagramChannel.open()) { channel.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); } } + @EntitlementTest(expectedAccess = PLUGINS) static void datagramChannelConnect() throws IOException { try (var channel = DatagramChannel.open()) { channel.configureBlocking(false); @@ -229,6 +259,7 @@ static void datagramChannelConnect() throws IOException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void datagramChannelSend() throws IOException { try (var channel = DatagramChannel.open()) { channel.configureBlocking(false); @@ -236,6 +267,7 @@ static void datagramChannelSend() throws IOException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void datagramChannelReceive() throws IOException { try (var channel = DatagramChannel.open()) { channel.configureBlocking(false); @@ -243,4 +275,149 @@ static void datagramChannelReceive() throws IOException { channel.receive(ByteBuffer.wrap(buffer)); } } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createURLStreamHandlerProvider() { + var x = new URLStreamHandlerProvider() { + @Override + public URLStreamHandler createURLStreamHandler(String protocol) { + return null; + } + }; + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createURLWithURLStreamHandler() throws MalformedURLException { + var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createURLWithURLStreamHandler2() throws MalformedURLException { + var x = new URL(null, "spec", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void setDefaultResponseCache() { + ResponseCache.setDefault(null); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void setDefaultProxySelector() { + ProxySelector.setDefault(null); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void setDefaultSSLContext() throws NoSuchAlgorithmException { + SSLContext.setDefault(SSLContext.getDefault()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void setDefaultHostnameVerifier() { + HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> false); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void setDefaultSSLSocketFactory() { + HttpsURLConnection.setDefaultSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void setHttpsConnectionProperties() { + new DummyImplementations.DummyHttpsURLConnection().setSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void datagramSocket$$setDatagramSocketImplFactory() throws IOException { + DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpURLConnection$$setFollowRedirects() { + HttpURLConnection.setFollowRedirects(HttpURLConnection.getFollowRedirects()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void serverSocket$$setSocketFactory() throws IOException { + ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void socket$$setSocketImplFactory() throws IOException { + Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void url$$setURLStreamHandlerFactory() { + URL.setURLStreamHandlerFactory(__ -> { throw new IllegalStateException(); }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void urlConnection$$setFileNameMap() { + URLConnection.setFileNameMap(__ -> { throw new IllegalStateException(); }); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void urlConnection$$setContentHandlerFactory() { + URLConnection.setContentHandlerFactory(__ -> { throw new IllegalStateException(); }); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void bindDatagramSocket() throws SocketException { + try (var socket = new DatagramSocket(null)) { + socket.bind(null); + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void connectDatagramSocket() throws SocketException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.connect(new InetSocketAddress(1234)); + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void joinGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.joinGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void leaveGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.leaveGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sendDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.send(new DatagramPacket(new byte[] { 0 }, 1, InetAddress.getLocalHost(), 1234)); + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void receiveDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.receive(new DatagramPacket(new byte[1], 1, InetAddress.getLocalHost(), 1234)); + } + } + + private NetworkAccessCheckActions() {} } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioChannelsActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioChannelsActions.java new file mode 100644 index 0000000000000..c8271b843f87a --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioChannelsActions.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; + +import java.io.FileDescriptor; +import java.io.IOException; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.util.Set; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class NioChannelsActions { + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createFileChannel() throws IOException { + new DummyImplementations.DummyFileChannel().close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileChannelOpenForWrite() throws IOException { + FileChannel.open(FileCheckActions.readWriteFile(), StandardOpenOption.WRITE).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileChannelOpenForRead() throws IOException { + FileChannel.open(FileCheckActions.readFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileChannelOpenForWriteWithOptions() throws IOException { + FileChannel.open(FileCheckActions.readWriteFile(), Set.of(StandardOpenOption.WRITE)).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void fileChannelOpenForReadWithOptions() throws IOException { + FileChannel.open(FileCheckActions.readFile(), Set.of(StandardOpenOption.READ)).close(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createAsynchronousFileChannel() { + new DummyImplementations.DummyAsynchronousFileChannel().close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void asynchronousFileChannelOpenForWrite() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + AsynchronousFileChannel.open(file, StandardOpenOption.WRITE).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void asynchronousFileChannelOpenForRead() throws IOException { + var file = EntitledActions.createTempFileForRead(); + AsynchronousFileChannel.open(file).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void asynchronousFileChannelOpenForWriteWithOptions() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + AsynchronousFileChannel.open(file, Set.of(StandardOpenOption.WRITE), EsExecutors.DIRECT_EXECUTOR_SERVICE).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void asynchronousFileChannelOpenForReadWithOptions() throws IOException { + var file = EntitledActions.createTempFileForRead(); + AsynchronousFileChannel.open(file, Set.of(StandardOpenOption.READ), EsExecutors.DIRECT_EXECUTOR_SERVICE).close(); + } + + @SuppressForbidden(reason = "specifically testing jdk.nio.Channels") + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void channelsReadWriteSelectableChannel() throws IOException { + jdk.nio.Channels.readWriteSelectableChannel(new FileDescriptor(), new DummyImplementations.DummySelectableChannelCloser()).close(); + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java new file mode 100644 index 0000000000000..09ecad2126efc --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFileSystemActions.java @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystemException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileOwnerAttributeView; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class NioFileSystemActions { + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void createFileSystemProvider() { + new DummyImplementations.DummyFileSystemProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkNewFileSystemFromUri() throws IOException { + try (var fs = FileSystems.getDefault().provider().newFileSystem(URI.create("/dummy/path"), Map.of())) {} + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkNewFileSystemFromPath() { + var fs = FileSystems.getDefault().provider(); + try (var newFs = fs.newFileSystem(Path.of("/dummy/path"), Map.of())) {} catch (IOException e) { + // When entitled, we expect to throw IOException, as the path is not valid - we don't really want to create a FS + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewInputStream() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var is = fs.newInputStream(FileCheckActions.readFile())) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewOutputStream() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var os = fs.newOutputStream(FileCheckActions.readWriteFile())) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewFileChannelRead() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var fc = fs.newFileChannel(FileCheckActions.readFile(), Set.of(StandardOpenOption.READ))) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewFileChannelWrite() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var fc = fs.newFileChannel(FileCheckActions.readWriteFile(), Set.of(StandardOpenOption.WRITE))) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewAsynchronousFileChannel() throws IOException { + var fs = FileSystems.getDefault().provider(); + try ( + var fc = fs.newAsynchronousFileChannel( + FileCheckActions.readWriteFile(), + Set.of(StandardOpenOption.WRITE), + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) + ) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewByteChannel() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var bc = fs.newByteChannel(FileCheckActions.readWriteFile(), Set.of(StandardOpenOption.WRITE))) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkNewDirectoryStream() throws IOException { + var fs = FileSystems.getDefault().provider(); + try (var bc = fs.newDirectoryStream(FileCheckActions.readDir(), entry -> false)) {} + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCreateDirectory() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + fs.createDirectory(directory.resolve("subdir")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCreateSymbolicLink() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + fs.createSymbolicLink(directory.resolve("link"), FileCheckActions.readFile()); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCreateLink() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + fs.createLink(directory.resolve("link"), FileCheckActions.readFile()); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkDelete() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForWrite(); + fs.delete(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkDeleteIfExists() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForWrite(); + fs.deleteIfExists(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadSymbolicLink() throws IOException { + var fs = FileSystems.getDefault().provider(); + var link = EntitledActions.createTempSymbolicLink(); + fs.readSymbolicLink(link); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCopy() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + fs.copy(FileCheckActions.readFile(), directory.resolve("copied")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkMove() throws IOException { + var fs = FileSystems.getDefault().provider(); + var directory = EntitledActions.createTempDirectoryForWrite(); + var file = EntitledActions.createTempFileForWrite(); + fs.move(file, directory.resolve("moved")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkIsSameFile() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.isSameFile(FileCheckActions.readWriteFile(), FileCheckActions.readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkIsHidden() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.isHidden(FileCheckActions.readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkGetFileStore() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForRead(); + var store = fs.getFileStore(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkCheckAccess() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.checkAccess(FileCheckActions.readFile()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkGetFileAttributeView() { + var fs = FileSystems.getDefault().provider(); + fs.getFileAttributeView(FileCheckActions.readFile(), FileOwnerAttributeView.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadAttributesWithClass() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.readAttributes(FileCheckActions.readFile(), BasicFileAttributes.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadAttributesWithString() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.readAttributes(FileCheckActions.readFile(), "*"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkSetAttribute() throws IOException { + var fs = FileSystems.getDefault().provider(); + var file = EntitledActions.createTempFileForWrite(); + try { + fs.setAttribute(file, "dos:hidden", true); + } catch (UnsupportedOperationException | IllegalArgumentException | FileSystemException e) { + // OK if the file does not have/does not support the attribute + } + } + + private NioFileSystemActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFilesActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFilesActions.java new file mode 100644 index 0000000000000..8f5ba8cadfb53 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NioFilesActions.java @@ -0,0 +1,481 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystemException; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileOwnerAttributeView; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.UserPrincipal; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; +import static org.elasticsearch.entitlement.qa.test.FileCheckActions.readDir; +import static org.elasticsearch.entitlement.qa.test.FileCheckActions.readFile; +import static org.elasticsearch.entitlement.qa.test.FileCheckActions.readWriteDir; +import static org.elasticsearch.entitlement.qa.test.FileCheckActions.readWriteFile; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class NioFilesActions { + + @EntitlementTest(expectedAccess = PLUGINS) + static void filesGetOwner() throws IOException { + Files.getOwner(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void filesProbeContentType() throws IOException { + Files.probeContentType(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void filesSetOwner() throws IOException { + UserPrincipal owner = EntitledActions.getFileOwner(readWriteFile()); + Files.setOwner(readWriteFile(), owner); // set to existing owner, just trying to execute the method + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewInputStream() throws IOException { + Files.newInputStream(readFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewOutputStream() throws IOException { + Files.newOutputStream(readWriteFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewByteChannelRead() throws IOException { + Files.newByteChannel(readFile(), Set.of(StandardOpenOption.READ)).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewByteChannelWrite() throws IOException { + Files.newByteChannel(readWriteFile(), Set.of(StandardOpenOption.WRITE)).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewByteChannelReadVarargs() throws IOException { + Files.newByteChannel(readFile(), StandardOpenOption.READ).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewByteChannelWriteVarargs() throws IOException { + Files.newByteChannel(readWriteFile(), StandardOpenOption.WRITE).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewDirectoryStream() throws IOException { + Files.newDirectoryStream(FileCheckActions.readDir()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewDirectoryStreamGlob() throws IOException { + Files.newDirectoryStream(FileCheckActions.readDir(), "*").close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewDirectoryStreamFilter() throws IOException { + Files.newDirectoryStream(FileCheckActions.readDir(), entry -> false).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateFile() throws IOException { + Files.createFile(readWriteDir().resolve("file.txt")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateDirectory() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.createDirectory(directory.resolve("subdir")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateDirectories() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.createDirectories(directory.resolve("subdir").resolve("subsubdir")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateTempFileInDir() throws IOException { + Files.createTempFile(readWriteDir(), "prefix", "suffix"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateTempDirectoryInDir() throws IOException { + Files.createTempDirectory(readWriteDir(), "prefix"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateSymbolicLink() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + Files.createSymbolicLink(directory.resolve("link"), readFile()); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateRelativeSymbolicLink() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + Files.createSymbolicLink(directory.resolve("link"), Path.of("target")); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateLink() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + try { + Files.createLink(directory.resolve("link"), readFile()); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCreateRelativeLink() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + var target = directory.resolve("target"); + try { + Files.createLink(directory.resolve("link"), Path.of("target")); + } catch (UnsupportedOperationException | FileSystemException e) { + // OK not to implement symbolic link in the filesystem + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesDelete() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + Files.delete(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesDeleteIfExists() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + Files.deleteIfExists(file); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadSymbolicLink() throws IOException { + var link = EntitledActions.createTempSymbolicLink(); + Files.readSymbolicLink(link); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCopy() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.copy(readFile(), directory.resolve("copied")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesMove() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + var file = EntitledActions.createTempFileForWrite(); + Files.move(file, directory.resolve("moved")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsSameFile() throws IOException { + Files.isSameFile(readWriteFile(), readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesMismatch() throws IOException { + Files.mismatch(readWriteFile(), readFile()); + } + + @SuppressForbidden(reason = "testing entitlements on this API specifically") + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsHidden() throws IOException { + Files.isHidden(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesGetFileStore() throws IOException { + var file = EntitledActions.createTempFileForRead(); + Files.getFileStore(file); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void checkFilesGetFileAttributeView() { + Files.getFileAttributeView(readFile(), FileOwnerAttributeView.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadAttributesWithClass() throws IOException { + Files.readAttributes(readFile(), BasicFileAttributes.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadAttributesWithString() throws IOException { + Files.readAttributes(readFile(), "*"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesGetAttribute() throws IOException { + try { + Files.getAttribute(readFile(), "dos:hidden"); + } catch (UnsupportedOperationException | IllegalArgumentException | FileSystemException e) { + // OK if the file does not have/does not support the attribute + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesSetAttribute() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + try { + Files.setAttribute(file, "dos:hidden", true); + } catch (UnsupportedOperationException | IllegalArgumentException | FileSystemException e) { + // OK if the file does not have/does not support the attribute + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesGetPosixFilePermissions() throws IOException { + try { + Files.getPosixFilePermissions(readFile()); + } catch (UnsupportedOperationException | IllegalArgumentException | FileSystemException e) { + // OK if the file does not have/does not support the attribute + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesSetPosixFilePermissions() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + try { + Files.setPosixFilePermissions(file, Set.of()); + } catch (UnsupportedOperationException | IllegalArgumentException | FileSystemException e) { + // OK if the file does not have/does not support the attribute + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsSymbolicLink() { + Files.isSymbolicLink(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsDirectory() { + Files.isDirectory(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsRegularFile() { + Files.isRegularFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesGetLastModifiedTime() throws IOException { + Files.getLastModifiedTime(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesSetLastModifiedTime() throws IOException { + var file = EntitledActions.createTempFileForWrite(); + Files.setLastModifiedTime(file, FileTime.from(Instant.now())); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesSize() throws IOException { + Files.size(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesExists() { + Files.exists(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNotExists() { + Files.notExists(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsReadable() { + Files.isReadable(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsWriteable() { + Files.isWritable(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesIsExecutable() { + Files.isExecutable(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWalkFileTree() throws IOException { + Files.walkFileTree(readDir(), dummyVisitor()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWalkFileTreeWithOptions() throws IOException { + Files.walkFileTree(readDir(), Set.of(FileVisitOption.FOLLOW_LINKS), 2, dummyVisitor()); + } + + private static FileVisitor dummyVisitor() { + return new FileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + return FileVisitResult.SKIP_SUBTREE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + return FileVisitResult.SKIP_SUBTREE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + return FileVisitResult.SKIP_SUBTREE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.SKIP_SUBTREE; + } + }; + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewBufferedReader() throws IOException { + Files.newBufferedReader(readFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewBufferedReaderWithCharset() throws IOException { + Files.newBufferedReader(readFile(), Charset.defaultCharset()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewBufferedWriter() throws IOException { + Files.newBufferedWriter(readWriteFile(), StandardOpenOption.WRITE).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesNewBufferedWriterWithCharset() throws IOException { + Files.newBufferedWriter(readWriteFile(), Charset.defaultCharset(), StandardOpenOption.WRITE).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCopyInputStream() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.copy(new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)), directory.resolve("copied")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesCopyOutputStream() throws IOException { + Files.copy(readFile(), new ByteArrayOutputStream()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadAllBytes() throws IOException { + Files.readAllBytes(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadString() throws IOException { + Files.readString(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadStringWithCharset() throws IOException { + Files.readString(readFile(), Charset.defaultCharset()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadAllLines() throws IOException { + Files.readAllLines(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesReadAllLinesWithCharset() throws IOException { + Files.readAllLines(readFile(), Charset.defaultCharset()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWrite() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.write(directory.resolve("file"), "foo".getBytes(StandardCharsets.UTF_8)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWriteLines() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.write(directory.resolve("file"), List.of("foo")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWriteString() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.writeString(directory.resolve("file"), "foo"); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWriteStringWithCharset() throws IOException { + var directory = EntitledActions.createTempDirectoryForWrite(); + Files.writeString(directory.resolve("file"), "foo", Charset.defaultCharset()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesList() throws IOException { + Files.list(readDir()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWalk() throws IOException { + Files.walk(readDir()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesWalkWithDepth() throws IOException { + Files.walk(readDir(), 2).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesFind() throws IOException { + Files.find(readDir(), 2, (path, basicFileAttributes) -> false).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesLines() throws IOException { + Files.lines(readFile()).close(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkFilesLinesWithCharset() throws IOException { + Files.lines(readFile(), Charset.defaultCharset()).close(); + } + + private NioFilesActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/OperatingSystemActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/OperatingSystemActions.java new file mode 100644 index 0000000000000..82bac3d6808eb --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/OperatingSystemActions.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import java.io.IOException; +import java.util.List; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class OperatingSystemActions { + + static void processBuilder_start() throws IOException { + new ProcessBuilder("").start(); + } + + static void processBuilder_startPipeline() throws IOException { + ProcessBuilder.startPipeline(List.of()); + } + + private OperatingSystemActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/PathActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/PathActions.java new file mode 100644 index 0000000000000..fa75395f62209 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/PathActions.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.util.Arrays; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressWarnings({ "unused" /* called via reflection */, "rawtypes" }) +class PathActions { + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkToRealPath() throws IOException { + FileCheckActions.readFile().toRealPath(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED, expectedExceptionIfDenied = NoSuchFileException.class) + static void checkToRealPathForInvalidTarget() throws IOException { + Path invalidLink = EntitledActions.createTempSymbolicLink(FileCheckActions.readDir().resolve("invalid")); + try { + EntitledActions.pathToRealPath(invalidLink); // throws NoSuchFileException when checking entitlements due to invalid target + } catch (NoSuchFileException e) { + assert Arrays.stream(e.getStackTrace()).anyMatch(t -> t.getClassName().equals(PolicyManager.class.getName())) + : "Expected NoSuchFileException to be thrown by entitlements check"; + throw e; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkToRealPathWithK8sLikeMount() throws IOException, Exception { + EntitledActions.createK8sLikeMount().toRealPath(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkToRealPathNoFollow() throws IOException { + FileCheckActions.readFile().toRealPath(LinkOption.NOFOLLOW_LINKS); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkRegister() throws IOException { + try (var watchService = FileSystems.getDefault().newWatchService()) { + FileCheckActions.readFile().register(watchService, new WatchEvent.Kind[0]); + } catch (IllegalArgumentException e) { + // intentionally no events registered + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkRegisterWithModifiers() throws IOException { + try (var watchService = FileSystems.getDefault().newWatchService()) { + FileCheckActions.readFile().register(watchService, new WatchEvent.Kind[0], new WatchEvent.Modifier[0]); + } catch (IllegalArgumentException e) { + // intentionally no events registered + } + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java index 9b8cae1b72d29..f27e75a10b321 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java @@ -11,20 +11,9 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.CheckedRunnable; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyBreakIteratorProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCalendarDataProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCalendarNameProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCollatorProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCurrencyNameProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyDateFormatProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyDateFormatSymbolsProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyDecimalFormatSymbolsProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyLocaleNameProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyLocaleServiceProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyNumberFormatProvider; -import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyTimeZoneNameProvider; +import org.elasticsearch.env.Environment; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.BaseRestHandler; @@ -32,450 +21,147 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; -import java.io.IOException; -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.HttpURLConnection; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.NetworkInterface; -import java.net.ProxySelector; -import java.net.ResponseCache; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketException; -import java.net.URL; -import java.net.URLClassLoader; -import java.net.URLConnection; -import java.net.URLStreamHandler; -import java.net.spi.URLStreamHandlerProvider; -import java.security.NoSuchAlgorithmException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; import static java.util.Map.entry; -import static org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction.CheckAction.alwaysDenied; -import static org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction.CheckAction.deniedToPlugins; -import static org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction.CheckAction.forPlugins; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_ALLOWED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; import static org.elasticsearch.rest.RestRequest.Method.GET; @SuppressWarnings("unused") public class RestEntitlementsCheckAction extends BaseRestHandler { private static final Logger logger = LogManager.getLogger(RestEntitlementsCheckAction.class); - public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing"); - - record CheckAction(CheckedRunnable action, boolean isAlwaysDeniedToPlugins, Integer fromJavaVersion) { - /** - * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case. - */ - static CheckAction deniedToPlugins(CheckedRunnable action) { - return new CheckAction(action, true, null); - } - - static CheckAction forPlugins(CheckedRunnable action) { - return new CheckAction(action, false, null); - } - static CheckAction alwaysDenied(CheckedRunnable action) { - return new CheckAction(action, true, null); + record CheckAction( + CheckedConsumer action, + EntitlementTest.ExpectedAccess expectedAccess, + Class expectedExceptionIfDenied, + Integer fromJavaVersion + ) {} + + private static final Map checkActions = collectTests( + FileCheckActions.class, + FileStoreActions.class, + JvmActions.class, + LoadNativeLibrariesCheckActions.class, + ManageThreadsActions.class, + NativeActions.class, + NetworkAccessCheckActions.class, + NioChannelsActions.class, + NioFilesActions.class, + NioFileSystemActions.class, + OperatingSystemActions.class, + PathActions.class, + SpiActions.class, + SystemActions.class, + URLConnectionFileActions.class, + URLConnectionNetworkActions.class, + VersionSpecificManageThreadsActions.class, + VersionSpecificNioFileSystemActions.class + ); + + private static Map collectTests(Class... testClasses) { + List> entries = new ArrayList<>(); + for (Class testClass : testClasses) { + getTestEntries(entries, testClass, a -> a.fromJavaVersion() == null || Runtime.version().feature() >= a.fromJavaVersion()); } + @SuppressWarnings({ "unchecked", "rawtypes" }) + Entry[] entriesArray = entries.toArray(new Entry[0]); + return Map.ofEntries(entriesArray); } - private static final Map checkActions = Stream.>of( - entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)), - entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)), - entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)), - entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)), - entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)), - entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline)), - entry("set_https_connection_properties", forPlugins(RestEntitlementsCheckAction::setHttpsConnectionProperties)), - entry("set_default_ssl_socket_factory", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLSocketFactory)), - entry("set_default_hostname_verifier", alwaysDenied(RestEntitlementsCheckAction::setDefaultHostnameVerifier)), - entry("set_default_ssl_context", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLContext)), - entry("system_setIn", alwaysDenied(RestEntitlementsCheckAction::system$$setIn)), - entry("system_setOut", alwaysDenied(RestEntitlementsCheckAction::system$$setOut)), - entry("system_setErr", alwaysDenied(RestEntitlementsCheckAction::system$$setErr)), - entry("runtime_addShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$addShutdownHook)), - entry("runtime_removeShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$$removeShutdownHook)), - entry( - "thread_setDefaultUncaughtExceptionHandler", - alwaysDenied(RestEntitlementsCheckAction::thread$$setDefaultUncaughtExceptionHandler) - ), - entry("localeServiceProvider", alwaysDenied(RestEntitlementsCheckAction::localeServiceProvider$)), - entry("breakIteratorProvider", alwaysDenied(RestEntitlementsCheckAction::breakIteratorProvider$)), - entry("collatorProvider", alwaysDenied(RestEntitlementsCheckAction::collatorProvider$)), - entry("dateFormatProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatProvider$)), - entry("dateFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatSymbolsProvider$)), - entry("decimalFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::decimalFormatSymbolsProvider$)), - entry("numberFormatProvider", alwaysDenied(RestEntitlementsCheckAction::numberFormatProvider$)), - entry("calendarDataProvider", alwaysDenied(RestEntitlementsCheckAction::calendarDataProvider$)), - entry("calendarNameProvider", alwaysDenied(RestEntitlementsCheckAction::calendarNameProvider$)), - entry("currencyNameProvider", alwaysDenied(RestEntitlementsCheckAction::currencyNameProvider$)), - entry("localeNameProvider", alwaysDenied(RestEntitlementsCheckAction::localeNameProvider$)), - entry("timeZoneNameProvider", alwaysDenied(RestEntitlementsCheckAction::timeZoneNameProvider$)), - entry("logManager", alwaysDenied(RestEntitlementsCheckAction::logManager$)), - - entry("locale_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultLocale)), - entry("locale_setDefaultForCategory", alwaysDenied(WritePropertiesCheckActions::setDefaultLocaleForCategory)), - entry("timeZone_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultTimeZone)), - - entry("system_setProperty", forPlugins(WritePropertiesCheckActions::setSystemProperty)), - entry("system_clearProperty", forPlugins(WritePropertiesCheckActions::clearSystemProperty)), - entry("system_setSystemProperties", alwaysDenied(WritePropertiesCheckActions::setSystemProperties)), - - // This group is a bit nasty: if entitlements don't prevent these, then networking is - // irreparably borked for the remainder of the test run. - entry( - "datagramSocket_setDatagramSocketImplFactory", - alwaysDenied(RestEntitlementsCheckAction::datagramSocket$$setDatagramSocketImplFactory) - ), - entry("httpURLConnection_setFollowRedirects", alwaysDenied(RestEntitlementsCheckAction::httpURLConnection$$setFollowRedirects)), - entry("serverSocket_setSocketFactory", alwaysDenied(RestEntitlementsCheckAction::serverSocket$$setSocketFactory)), - entry("socket_setSocketImplFactory", alwaysDenied(RestEntitlementsCheckAction::socket$$setSocketImplFactory)), - entry("url_setURLStreamHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::url$$setURLStreamHandlerFactory)), - entry("urlConnection_setFileNameMap", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setFileNameMap)), - entry("urlConnection_setContentHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setContentHandlerFactory)), + private final Environment environment; - entry("proxySelector_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultProxySelector)), - entry("responseCache_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultResponseCache)), - entry( - "createInetAddressResolverProvider", - new CheckAction(VersionSpecificNetworkChecks::createInetAddressResolverProvider, true, 18) - ), - entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)), - entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)), - entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)), - entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)), - entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)), - entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)), - entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)), - entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)), - entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket)), - - entry("create_socket_with_proxy", forPlugins(NetworkAccessCheckActions::createSocketWithProxy)), - entry("socket_bind", forPlugins(NetworkAccessCheckActions::socketBind)), - entry("socket_connect", forPlugins(NetworkAccessCheckActions::socketConnect)), - entry("server_socket_bind", forPlugins(NetworkAccessCheckActions::serverSocketBind)), - entry("server_socket_accept", forPlugins(NetworkAccessCheckActions::serverSocketAccept)), - - entry("url_open_connection_proxy", forPlugins(NetworkAccessCheckActions::urlOpenConnectionWithProxy)), - entry("http_client_send", forPlugins(VersionSpecificNetworkChecks::httpClientSend)), - entry("http_client_send_async", forPlugins(VersionSpecificNetworkChecks::httpClientSendAsync)), - entry("create_ldap_cert_store", forPlugins(NetworkAccessCheckActions::createLDAPCertStore)), - - entry("server_socket_channel_bind", forPlugins(NetworkAccessCheckActions::serverSocketChannelBind)), - entry("server_socket_channel_bind_backlog", forPlugins(NetworkAccessCheckActions::serverSocketChannelBindWithBacklog)), - entry("server_socket_channel_accept", forPlugins(NetworkAccessCheckActions::serverSocketChannelAccept)), - entry("asynchronous_server_socket_channel_bind", forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelBind)), - entry( - "asynchronous_server_socket_channel_bind_backlog", - forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelBindWithBacklog) - ), - entry("asynchronous_server_socket_channel_accept", forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelAccept)), - entry( - "asynchronous_server_socket_channel_accept_with_handler", - forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelAcceptWithHandler) - ), - entry("socket_channel_bind", forPlugins(NetworkAccessCheckActions::socketChannelBind)), - entry("socket_channel_connect", forPlugins(NetworkAccessCheckActions::socketChannelConnect)), - entry("asynchronous_socket_channel_bind", forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelBind)), - entry("asynchronous_socket_channel_connect", forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelConnect)), - entry( - "asynchronous_socket_channel_connect_with_completion", - forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelConnectWithCompletion) - ), - entry("datagram_channel_bind", forPlugins(NetworkAccessCheckActions::datagramChannelBind)), - entry("datagram_channel_connect", forPlugins(NetworkAccessCheckActions::datagramChannelConnect)), - entry("datagram_channel_send", forPlugins(NetworkAccessCheckActions::datagramChannelSend)), - entry("datagram_channel_receive", forPlugins(NetworkAccessCheckActions::datagramChannelReceive)), - - entry("runtime_load", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoad)), - entry("runtime_load_library", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoadLibrary)), - entry("system_load", forPlugins(LoadNativeLibrariesCheckActions::systemLoad)), - entry("system_load_library", forPlugins(LoadNativeLibrariesCheckActions::systemLoadLibrary)), - entry("enable_native_access", new CheckAction(VersionSpecificNativeChecks::enableNativeAccess, false, 22)), - entry("address_target_layout", new CheckAction(VersionSpecificNativeChecks::addressLayoutWithTargetLayout, false, 22)), - entry("donwncall_handle", new CheckAction(VersionSpecificNativeChecks::linkerDowncallHandle, false, 22)), - entry("donwncall_handle_with_address", new CheckAction(VersionSpecificNativeChecks::linkerDowncallHandleWithAddress, false, 22)), - entry("upcall_stub", new CheckAction(VersionSpecificNativeChecks::linkerUpcallStub, false, 22)), - entry("reinterpret", new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpret, false, 22)), - entry("reinterpret_cleanup", new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpretWithCleanup, false, 22)), - entry( - "reinterpret_size_cleanup", - new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpretWithSizeAndCleanup, false, 22) - ), - entry("symbol_lookup_name", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithName, false, 22)), - entry("symbol_lookup_path", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithPath, false, 22)), - entry("create_scanner", forPlugins(FileCheckActions::createScannerFile)), - entry("create_scanner_with_charset", forPlugins(FileCheckActions::createScannerFileWithCharset)), - entry("create_scanner_with_charset_name", forPlugins(FileCheckActions::createScannerFileWithCharsetName)), - entry("create_file_output_stream_string", forPlugins(FileCheckActions::createFileOutputStreamString)), - entry("create_file_output_stream_string_with_append", forPlugins(FileCheckActions::createFileOutputStreamStringWithAppend)), - entry("create_file_output_stream_file", forPlugins(FileCheckActions::createFileOutputStreamFile)), - entry("create_file_output_stream_file_with_append", forPlugins(FileCheckActions::createFileOutputStreamFileWithAppend)), - entry("files_probe_content_type", forPlugins(FileCheckActions::filesProbeContentType)), - entry("files_set_owner", forPlugins(FileCheckActions::filesSetOwner)) - ) - .filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion()) - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); - - private static void createURLStreamHandlerProvider() { - var x = new URLStreamHandlerProvider() { - @Override - public URLStreamHandler createURLStreamHandler(String protocol) { - return null; - } - }; + public RestEntitlementsCheckAction(Environment environment) { + this.environment = environment; } - @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler() throws MalformedURLException { - var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); + @SuppressForbidden(reason = "Need package private methods so we don't have to make them all public") + private static Method[] getDeclaredMethods(Class clazz) { + return clazz.getDeclaredMethods(); } - @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler2() throws MalformedURLException { - var x = new URL(null, "spec", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; + private static void getTestEntries(List> entries, Class actionsClass, Predicate filter) { + for (var method : getDeclaredMethods(actionsClass)) { + var testAnnotation = method.getAnnotation(EntitlementTest.class); + if (testAnnotation == null) { + continue; } - }); - } - - private static void setDefaultResponseCache() { - ResponseCache.setDefault(null); - } - - private static void setDefaultProxySelector() { - ProxySelector.setDefault(null); - } - - private static void setDefaultSSLContext() throws NoSuchAlgorithmException { - SSLContext.setDefault(SSLContext.getDefault()); - } - - private static void setDefaultHostnameVerifier() { - HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> false); - } - - private static void setDefaultSSLSocketFactory() { - HttpsURLConnection.setDefaultSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory()); - } - - @SuppressForbidden(reason = "Specifically testing Runtime.exit") - private static void runtimeExit() { - Runtime.getRuntime().exit(123); - } - - @SuppressForbidden(reason = "Specifically testing Runtime.halt") - private static void runtimeHalt() { - Runtime.getRuntime().halt(123); - } - - @SuppressForbidden(reason = "Specifically testing System.exit") - private static void systemExit() { - System.exit(123); - } - - private static void createClassLoader() throws IOException { - try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) { - logger.info("Created URLClassLoader [{}]", classLoader.getName()); - } - } - - private static void processBuilder_start() throws IOException { - new ProcessBuilder("").start(); - } - - private static void processBuilder_startPipeline() throws IOException { - ProcessBuilder.startPipeline(List.of()); - } - - private static void setHttpsConnectionProperties() { - new DummyImplementations.DummyHttpsURLConnection().setSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory()); - } - - private static void system$$setIn() { - System.setIn(System.in); - } - - @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams") - private static void system$$setOut() { - System.setOut(System.out); - } - - @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams") - private static void system$$setErr() { - System.setErr(System.err); - } - - private static void runtime$addShutdownHook() { - Runtime.getRuntime().addShutdownHook(NO_OP_SHUTDOWN_HOOK); - } - - private static void runtime$$removeShutdownHook() { - Runtime.getRuntime().removeShutdownHook(NO_OP_SHUTDOWN_HOOK); - } - - private static void thread$$setDefaultUncaughtExceptionHandler() { - Thread.setDefaultUncaughtExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()); - } - - private static void localeServiceProvider$() { - new DummyLocaleServiceProvider(); - } - - private static void breakIteratorProvider$() { - new DummyBreakIteratorProvider(); - } - - private static void collatorProvider$() { - new DummyCollatorProvider(); - } - - private static void dateFormatProvider$() { - new DummyDateFormatProvider(); - } - - private static void dateFormatSymbolsProvider$() { - new DummyDateFormatSymbolsProvider(); - } - - private static void decimalFormatSymbolsProvider$() { - new DummyDecimalFormatSymbolsProvider(); - } - - private static void numberFormatProvider$() { - new DummyNumberFormatProvider(); - } - - private static void calendarDataProvider$() { - new DummyCalendarDataProvider(); - } - - private static void calendarNameProvider$() { - new DummyCalendarNameProvider(); - } - - private static void currencyNameProvider$() { - new DummyCurrencyNameProvider(); - } - - private static void localeNameProvider$() { - new DummyLocaleNameProvider(); - } - - private static void timeZoneNameProvider$() { - new DummyTimeZoneNameProvider(); - } - - private static void logManager$() { - new java.util.logging.LogManager() { - }; - } - - @SuppressWarnings("deprecation") - @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void datagramSocket$$setDatagramSocketImplFactory() throws IOException { - DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); - } - - private static void httpURLConnection$$setFollowRedirects() { - HttpURLConnection.setFollowRedirects(HttpURLConnection.getFollowRedirects()); - } - - @SuppressWarnings("deprecation") - @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void serverSocket$$setSocketFactory() throws IOException { - ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); - } - - @SuppressWarnings("deprecation") - @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void socket$$setSocketImplFactory() throws IOException { - Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); - } - - private static void url$$setURLStreamHandlerFactory() { - URL.setURLStreamHandlerFactory(__ -> { throw new IllegalStateException(); }); - } - - private static void urlConnection$$setFileNameMap() { - URLConnection.setFileNameMap(__ -> { throw new IllegalStateException(); }); - } - - private static void urlConnection$$setContentHandlerFactory() { - URLConnection.setContentHandlerFactory(__ -> { throw new IllegalStateException(); }); - } - - private static void bindDatagramSocket() throws SocketException { - try (var socket = new DatagramSocket(null)) { - socket.bind(null); - } - } - - @SuppressForbidden(reason = "testing entitlements") - private static void connectDatagramSocket() throws SocketException { - try (var socket = new DummyImplementations.DummyDatagramSocket()) { - socket.connect(new InetSocketAddress(1234)); - } - } - - private static void joinGroupDatagramSocket() throws IOException { - try (var socket = new DummyImplementations.DummyDatagramSocket()) { - socket.joinGroup( - new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), - NetworkInterface.getByIndex(0) + if (Modifier.isStatic(method.getModifiers()) == false) { + throw new AssertionError("Entitlement test method [" + method + "] must be static"); + } + if (Modifier.isPrivate(method.getModifiers())) { + throw new AssertionError("Entitlement test method [" + method + "] must not be private"); + } + final CheckedConsumer call = createConsumerForMethod(method); + CheckedConsumer runnable = env -> { + try { + call.accept(env); + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } catch (InvocationTargetException e) { + if (e.getCause() instanceof Exception exc) { + throw exc; + } else { + throw new AssertionError(e); + } + } + }; + Integer fromJavaVersion = testAnnotation.fromJavaVersion() == -1 ? null : testAnnotation.fromJavaVersion(); + var checkAction = new CheckAction( + runnable, + testAnnotation.expectedAccess(), + testAnnotation.expectedExceptionIfDenied(), + fromJavaVersion ); + if (filter.test(checkAction)) { + entries.add(entry(method.getName(), checkAction)); + } } } - private static void leaveGroupDatagramSocket() throws IOException { - try (var socket = new DummyImplementations.DummyDatagramSocket()) { - socket.leaveGroup( - new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), - NetworkInterface.getByIndex(0) - ); + private static CheckedConsumer createConsumerForMethod(Method method) { + Class[] parameters = method.getParameterTypes(); + if (parameters.length == 0) { + return env -> method.invoke(null); } - } - - @SuppressForbidden(reason = "testing entitlements") - private static void sendDatagramSocket() throws IOException { - try (var socket = new DummyImplementations.DummyDatagramSocket()) { - socket.send(new DatagramPacket(new byte[] { 0 }, 1, InetAddress.getLocalHost(), 1234)); + if (parameters.length == 1 && parameters[0].equals(Environment.class)) { + return env -> method.invoke(null, env); } + throw new AssertionError("Entitlement test method [" + method + "] must have no parameters or 1 parameter (Environment)"); } - @SuppressForbidden(reason = "testing entitlements") - private static void receiveDatagramSocket() throws IOException { - try (var socket = new DummyImplementations.DummyDatagramSocket()) { - socket.receive(new DatagramPacket(new byte[1], 1, InetAddress.getLocalHost(), 1234)); - } + public static Set getCheckActionsAllowedInPlugins() { + return checkActions.entrySet() + .stream() + .filter(kv -> kv.getValue().expectedAccess().equals(PLUGINS) || kv.getValue().expectedAccess().equals(ALWAYS_ALLOWED)) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); } - public static Set getCheckActionsAllowedInPlugins() { + public static Set getAlwaysAllowedCheckActions() { return checkActions.entrySet() .stream() - .filter(kv -> kv.getValue().isAlwaysDeniedToPlugins() == false) + .filter(kv -> kv.getValue().expectedAccess().equals(ALWAYS_ALLOWED)) .map(Map.Entry::getKey) .collect(Collectors.toSet()); } - public static Set getAllCheckActions() { - return checkActions.keySet(); + public static Set getDeniableCheckActions() { + return checkActions.entrySet() + .stream() + .filter(kv -> kv.getValue().expectedAccess().equals(ALWAYS_ALLOWED) == false) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); } @Override @@ -502,8 +188,19 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli return channel -> { logger.info("Calling check action [{}]", actionName); - checkAction.action().run(); - channel.sendResponse(new RestResponse(RestStatus.OK, Strings.format("Succesfully executed action [%s]", actionName))); + RestResponse response; + try { + checkAction.action().accept(environment); + response = new RestResponse(RestStatus.OK, Strings.format("Succesfully executed action [%s]", actionName)); + } catch (Exception e) { + var statusCode = checkAction.expectedExceptionIfDenied.isInstance(e) + ? RestStatus.FORBIDDEN + : RestStatus.INTERNAL_SERVER_ERROR; + response = new RestResponse(channel, statusCode, e); + response.addHeader("expectedException", checkAction.expectedExceptionIfDenied.getName()); + } + logger.debug("Check action [{}] returned status [{}]", actionName, response.status().getStatus()); + channel.sendResponse(response); }; } } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SpiActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SpiActions.java new file mode 100644 index 0000000000000..aeb548c18fb69 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SpiActions.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import java.io.IOException; +import java.nio.channels.Channel; +import java.nio.channels.spi.SelectorProvider; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class SpiActions { + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createBreakIteratorProvider() { + new DummyImplementations.DummyBreakIteratorProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createCollatorProvider() { + new DummyImplementations.DummyCollatorProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createDateFormatProvider() { + new DummyImplementations.DummyDateFormatProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createDateFormatSymbolsProvider() { + new DummyImplementations.DummyDateFormatSymbolsProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createDecimalFormatSymbolsProvider() { + new DummyImplementations.DummyDecimalFormatSymbolsProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createNumberFormatProvider() { + new DummyImplementations.DummyNumberFormatProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createCalendarDataProvider() { + new DummyImplementations.DummyCalendarDataProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createCalendarNameProvider() { + new DummyImplementations.DummyCalendarNameProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createCurrencyNameProvider() { + new DummyImplementations.DummyCurrencyNameProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createLocaleNameProvider() { + new DummyImplementations.DummyLocaleNameProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createTimeZoneNameProvider() { + new DummyImplementations.DummyTimeZoneNameProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createLocaleServiceProvider() { + new DummyImplementations.DummyLocaleServiceProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void getInheritedChannel() throws IOException { + try (Channel channel = SelectorProvider.provider().inheritedChannel()) {} + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createSelectorProvider() { + new DummyImplementations.DummySelectorProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createAsynchronousChannelProvider() { + new DummyImplementations.DummyAsynchronousChannelProvider(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void createCharsetProvider() { + new DummyImplementations.DummyCharsetProvider(); + } + + private SpiActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SystemActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SystemActions.java new file mode 100644 index 0000000000000..c1281f50365a9 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SystemActions.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.SuppressForbidden; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + +@SuppressWarnings({ "unused" /* called via reflection */ }) +class SystemActions { + + @SuppressForbidden(reason = "Specifically testing Runtime.exit") + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void runtimeExit() { + Runtime.getRuntime().exit(123); + } + + @SuppressForbidden(reason = "Specifically testing Runtime.halt") + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void runtimeHalt() { + Runtime.getRuntime().halt(123); + } + + @SuppressForbidden(reason = "Specifically testing System.exit") + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void systemExit() { + System.exit(123); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void systemSetIn() { + System.setIn(System.in); + } + + @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams") + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void systemSetOut() { + System.setOut(System.out); + } + + @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams") + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void systemSetErr() { + System.setErr(System.err); + } + + private static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing"); + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void runtimeAddShutdownHook() { + Runtime.getRuntime().addShutdownHook(NO_OP_SHUTDOWN_HOOK); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void runtimeRemoveShutdownHook() { + Runtime.getRuntime().removeShutdownHook(NO_OP_SHUTDOWN_HOOK); + } + + private SystemActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/URLConnectionFileActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/URLConnectionFileActions.java new file mode 100644 index 0000000000000..6638a9659749c --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/URLConnectionFileActions.java @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; + +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URLConnection; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +class URLConnectionFileActions { + + private static void withJdkFileConnection(CheckedConsumer connectionConsumer) throws Exception { + var conn = EntitledActions.createFileURLConnection(); + // Be sure we got the connection implementation we want + assert conn.getClass().getSimpleName().equals("FileURLConnection"); + try { + connectionConsumer.accept(conn); + } catch (IOException e) { + // It's OK, it means we passed entitlement checks, and we tried to perform some operation + } + } + + private static void withJarConnection(CheckedConsumer connectionConsumer) throws Exception { + var conn = EntitledActions.createJarURLConnection(); + // Be sure we got the connection implementation we want + assert JarURLConnection.class.isAssignableFrom(conn.getClass()); + connectionConsumer.accept((JarURLConnection) conn); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionConnect() throws Exception { + withJdkFileConnection(URLConnection::connect); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetHeaderFields() throws Exception { + withJdkFileConnection(URLConnection::getHeaderFields); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetHeaderFieldWithName() throws Exception { + withJdkFileConnection(urlConnection -> urlConnection.getHeaderField("date")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetHeaderFieldWithIndex() throws Exception { + withJdkFileConnection(urlConnection -> urlConnection.getHeaderField(0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetContentLength() throws Exception { + withJdkFileConnection(URLConnection::getContentLength); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetContentLengthLong() throws Exception { + withJdkFileConnection(URLConnection::getContentLengthLong); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetHeaderFieldKey() throws Exception { + withJdkFileConnection(urlConnection -> urlConnection.getHeaderFieldKey(0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetLastModified() throws Exception { + withJdkFileConnection(URLConnection::getLastModified); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetInputStream() throws Exception { + withJdkFileConnection(URLConnection::getInputStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetContentType() throws Exception { + withJdkFileConnection(URLConnection::getContentType); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetContentEncoding() throws Exception { + withJdkFileConnection(URLConnection::getContentEncoding); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetExpiration() throws Exception { + withJdkFileConnection(URLConnection::getExpiration); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetDate() throws Exception { + withJdkFileConnection(URLConnection::getDate); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetHeaderFieldInt() throws Exception { + withJdkFileConnection(conn -> conn.getHeaderFieldInt("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetHeaderFieldLong() throws Exception { + withJdkFileConnection(conn -> conn.getHeaderFieldLong("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetContent() throws Exception { + withJdkFileConnection(URLConnection::getContent); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFileURLConnectionGetContentWithClasses() throws Exception { + withJdkFileConnection(conn -> conn.getContent(new Class[] { String.class })); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetManifest() throws Exception { + withJarConnection(JarURLConnection::getManifest); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetJarEntry() throws Exception { + withJarConnection(JarURLConnection::getJarEntry); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetAttributes() throws Exception { + withJarConnection(JarURLConnection::getAttributes); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetMainAttributes() throws Exception { + withJarConnection(JarURLConnection::getMainAttributes); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetCertificates() throws Exception { + withJarConnection(JarURLConnection::getCertificates); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetJarFile() throws Exception { + withJarConnection(JarURLConnection::getJarFile); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetJarEntry() throws Exception { + withJarConnection(JarURLConnection::getJarEntry); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionConnect() throws Exception { + withJarConnection(JarURLConnection::connect); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetInputStream() throws Exception { + withJarConnection(JarURLConnection::getInputStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetContentLength() throws Exception { + withJarConnection(JarURLConnection::getContentLength); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetContentLengthLong() throws Exception { + withJarConnection(JarURLConnection::getContentLengthLong); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetContent() throws Exception { + withJarConnection(JarURLConnection::getContent); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetContentType() throws Exception { + withJarConnection(JarURLConnection::getContentType); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunJarURLConnectionGetHeaderFieldWithName() throws Exception { + withJarConnection(conn -> conn.getHeaderField("field")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetContentEncoding() throws Exception { + withJarConnection(URLConnection::getContentEncoding); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetExpiration() throws Exception { + withJarConnection(URLConnection::getExpiration); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetDate() throws Exception { + withJarConnection(URLConnection::getDate); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetLastModified() throws Exception { + withJarConnection(URLConnection::getLastModified); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetHeaderFieldInt() throws Exception { + withJarConnection(conn -> conn.getHeaderFieldInt("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetHeaderFieldLong() throws Exception { + withJarConnection(conn -> conn.getHeaderFieldLong("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetHeaderFieldDate() throws Exception { + withJarConnection(conn -> conn.getHeaderFieldDate("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void netJarURLConnectionGetContent() throws Exception { + withJarConnection(conn -> conn.getContent(new Class[] { String.class })); + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/URLConnectionNetworkActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/URLConnectionNetworkActions.java new file mode 100644 index 0000000000000..95f8b6d3de748 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/URLConnectionNetworkActions.java @@ -0,0 +1,453 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.qa.entitled.EntitledActions; + +import java.io.IOException; +import java.io.InputStream; +import java.net.ConnectException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.Proxy; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLConnection; + +import javax.net.ssl.HttpsURLConnection; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressWarnings("unused") // everything is called via reflection +class URLConnectionNetworkActions { + + private static final URL HTTP_URL; + + static { + try { + HTTP_URL = URI.create("http://127.0.0.1/").toURL(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private static void withPlainNetworkConnection(CheckedConsumer connectionConsumer) throws Exception { + // Create a HttpURLConnection with minimal overrides to test calling directly into URLConnection methods as much as possible + var conn = new HttpURLConnection(HTTP_URL) { + @Override + public void connect() {} + + @Override + public void disconnect() {} + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public InputStream getInputStream() throws IOException { + // Mock an attempt to call connect + throw new ConnectException(); + } + }; + + try { + connectionConsumer.accept(conn); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + private static void withJdkHttpConnection(CheckedConsumer connectionConsumer) throws Exception { + var conn = EntitledActions.createHttpURLConnection(); + // Be sure we got the connection implementation we want + assert HttpURLConnection.class.isAssignableFrom(conn.getClass()); + try { + connectionConsumer.accept((HttpURLConnection) conn); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + private static void withJdkHttpsConnection(CheckedConsumer connectionConsumer) throws Exception { + var conn = EntitledActions.createHttpsURLConnection(); + // Be sure we got the connection implementation we want + assert HttpsURLConnection.class.isAssignableFrom(conn.getClass()); + try { + connectionConsumer.accept((HttpsURLConnection) conn); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + private static void withJdkFtpConnection(CheckedConsumer connectionConsumer) throws Exception { + var conn = EntitledActions.createFtpURLConnection(); + // Be sure we got the connection implementation we want + assert conn.getClass().getSimpleName().equals("FtpURLConnection"); + try { + connectionConsumer.accept(conn); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + private static void withJdkMailToConnection(CheckedConsumer connectionConsumer) throws Exception { + var conn = EntitledActions.createMailToURLConnection(); + // Be sure we got the connection implementation we want + assert conn.getClass().getSimpleName().equals("MailToURLConnection"); + try { + connectionConsumer.accept(conn); + } catch (IOException e) { + // It's OK, it means we passed entitlement checks, and we tried to perform some IO + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void urlOpenConnection() throws Exception { + URI.create("http://127.0.0.1:12345/").toURL().openConnection(); + } + + @EntitlementTest(expectedAccess = PLUGINS) + @SuppressForbidden(reason = "just testing, not a real connection") + static void urlOpenConnectionWithProxy() throws URISyntaxException, IOException { + var url = new URI("http://localhost").toURL(); + var urlConnection = url.openConnection(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(0))); + assert urlConnection != null; + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void urlOpenStream() throws Exception { + try { + URI.create("http://127.0.0.1:12345/").toURL().openStream().close(); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void urlGetContent() throws Exception { + try { + URI.create("http://127.0.0.1:12345/").toURL().getContent(); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void urlGetContentWithClasses() throws Exception { + try { + URI.create("http://127.0.0.1:12345/").toURL().getContent(new Class[] { String.class }); + } catch (java.net.ConnectException e) { + // It's OK, it means we passed entitlement checks, and we tried to connect + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetContentLength() throws Exception { + withPlainNetworkConnection(URLConnection::getContentLength); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetContentLength() throws Exception { + withJdkHttpConnection(URLConnection::getContentLength); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetContentType() throws Exception { + withPlainNetworkConnection(URLConnection::getContentType); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetContentType() throws Exception { + withJdkHttpConnection(URLConnection::getContentType); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetContentEncoding() throws Exception { + withPlainNetworkConnection(URLConnection::getContentEncoding); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetContentEncoding() throws Exception { + withJdkHttpConnection(URLConnection::getContentEncoding); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetExpiration() throws Exception { + withPlainNetworkConnection(URLConnection::getExpiration); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetExpiration() throws Exception { + withJdkHttpConnection(URLConnection::getExpiration); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetDate() throws Exception { + withPlainNetworkConnection(URLConnection::getDate); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetDate() throws Exception { + withJdkHttpConnection(URLConnection::getDate); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetLastModified() throws Exception { + withPlainNetworkConnection(URLConnection::getLastModified); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetLastModified() throws Exception { + withJdkHttpConnection(URLConnection::getLastModified); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetHeaderFieldInt() throws Exception { + withPlainNetworkConnection(conn -> conn.getHeaderFieldInt("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetHeaderFieldInt() throws Exception { + withJdkHttpConnection(conn -> conn.getHeaderFieldInt("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetHeaderFieldLong() throws Exception { + withPlainNetworkConnection(conn -> conn.getHeaderFieldLong("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetHeaderFieldLong() throws Exception { + withJdkHttpConnection(conn -> conn.getHeaderFieldLong("field", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetContent() throws Exception { + withPlainNetworkConnection(URLConnection::getContent); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetContent() throws Exception { + withJdkHttpConnection(URLConnection::getContent); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseUrlConnectionGetContentWithClasses() throws Exception { + withPlainNetworkConnection(conn -> conn.getContent(new Class[] { String.class })); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpConnectionGetContentWithClasses() throws Exception { + withJdkHttpConnection(conn -> conn.getContent(new Class[] { String.class })); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFtpURLConnectionConnect() throws Exception { + withJdkFtpConnection(URLConnection::connect); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFtpURLConnectionGetInputStream() throws Exception { + withJdkFtpConnection(URLConnection::getInputStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunFtpURLConnectionGetOutputStream() throws Exception { + withJdkFtpConnection(URLConnection::getOutputStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseHttpURLConnectionGetResponseCode() throws Exception { + withPlainNetworkConnection(HttpURLConnection::getResponseCode); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseHttpURLConnectionGetResponseMessage() throws Exception { + withPlainNetworkConnection(HttpURLConnection::getResponseMessage); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void baseHttpURLConnectionGetHeaderFieldDate() throws Exception { + withPlainNetworkConnection(conn -> conn.getHeaderFieldDate("date", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionConnect() throws Exception { + withJdkHttpConnection(HttpURLConnection::connect); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetOutputStream() throws Exception { + withJdkHttpConnection(httpURLConnection -> { + httpURLConnection.setDoOutput(true); + httpURLConnection.getOutputStream(); + }); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetInputStream() throws Exception { + withJdkHttpConnection(HttpURLConnection::getInputStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetErrorStream() throws Exception { + withJdkHttpConnection(HttpURLConnection::getErrorStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetHeaderFieldWithName() throws Exception { + withJdkHttpConnection(conn -> conn.getHeaderField("date")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetHeaderFields() throws Exception { + withJdkHttpConnection(HttpURLConnection::getHeaderFields); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetHeaderFieldWithIndex() throws Exception { + withJdkHttpConnection(conn -> conn.getHeaderField(0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpURLConnectionGetHeaderFieldKey() throws Exception { + withJdkHttpConnection(conn -> conn.getHeaderFieldKey(0)); + } + + // https + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplConnect() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::connect); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetOutputStream() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> { + httpsURLConnection.setDoOutput(true); + httpsURLConnection.getOutputStream(); + }); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetInputStream() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getInputStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetErrorStream() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getErrorStream); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFieldWithName() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getHeaderField("date")); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFields() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getHeaderFields); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFieldWithIndex() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getHeaderField(0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFieldKey() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getHeaderFieldKey(0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetResponseCode() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getResponseCode); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetResponseMessage() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getResponseMessage); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetContentLength() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getContentLength); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImpl$getContentLengthLong() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getContentLengthLong); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetContentType() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getContentType); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetContentEncoding() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getContentEncoding); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetExpiration() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getExpiration); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetDate() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getDate); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetLastModified() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getLastModified); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFieldInt() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getHeaderFieldInt("content-length", -1)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFieldLong() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getHeaderFieldLong("content-length", -1)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetHeaderFieldDate() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getHeaderFieldDate("date", 0)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetContent() throws Exception { + withJdkHttpsConnection(HttpsURLConnection::getContent); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunHttpsURLConnectionImplGetContentWithClasses() throws Exception { + withJdkHttpsConnection(httpsURLConnection -> httpsURLConnection.getContent(new Class[] { String.class })); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunMailToURLConnectionConnect() throws Exception { + withJdkMailToConnection(URLConnection::connect); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void sunMailToURLConnectionGetOutputStream() throws Exception { + withJdkMailToConnection(URLConnection::getOutputStream); + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificManageThreadsActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificManageThreadsActions.java new file mode 100644 index 0000000000000..12849ed20b7d9 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificManageThreadsActions.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.SuppressForbidden; + +@SuppressForbidden(reason = "testing entitlements") +@SuppressWarnings("unused") // used via reflection +class VersionSpecificManageThreadsActions { + private VersionSpecificManageThreadsActions() {} +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNativeChecks.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNativeChecks.java deleted file mode 100644 index cb84c9bd9042d..0000000000000 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNativeChecks.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.entitlement.qa.test; - -class VersionSpecificNativeChecks { - - static void enableNativeAccess() throws Exception {} - - static void addressLayoutWithTargetLayout() {} - - static void linkerDowncallHandle() {} - - static void linkerDowncallHandleWithAddress() {} - - static void linkerUpcallStub() throws NoSuchMethodException {} - - static void memorySegmentReinterpret() {} - - static void memorySegmentReinterpretWithCleanup() {} - - static void memorySegmentReinterpretWithSizeAndCleanup() {} - - static void symbolLookupWithPath() {} - - static void symbolLookupWithName() {} -} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java index 548bce8e2f766..2c06fb744a1bd 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java @@ -15,9 +15,13 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressWarnings({ "unused" /* called via reflection */ }) class VersionSpecificNetworkChecks { static void createInetAddressResolverProvider() {} + @EntitlementTest(expectedAccess = PLUGINS) static void httpClientSend() throws InterruptedException { HttpClient httpClient = HttpClient.newBuilder().build(); try { @@ -27,6 +31,7 @@ static void httpClientSend() throws InterruptedException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void httpClientSendAsync() { HttpClient httpClient = HttpClient.newBuilder().build(); httpClient.sendAsync(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding()); diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNioFileSystemActions.java similarity index 83% rename from qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy rename to libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNioFileSystemActions.java index fe1c607f2f196..5e7025916312b 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNioFileSystemActions.java @@ -7,7 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -grant { - // an unresolved permission - permission org.fake.FakePermission "fakeName"; -}; +package org.elasticsearch.entitlement.qa.test; + +class VersionSpecificNioFileSystemActions {} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main18/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java b/libs/entitlement/qa/entitlement-test-plugin/src/main18/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java index 5a456c65d8206..28c0d6ea55f05 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main18/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main18/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNetworkChecks.java @@ -17,7 +17,11 @@ import java.net.spi.InetAddressResolver; import java.net.spi.InetAddressResolverProvider; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + class VersionSpecificNetworkChecks { + @EntitlementTest(expectedAccess = SERVER_ONLY, fromJavaVersion = 18) static void createInetAddressResolverProvider() { var x = new InetAddressResolverProvider() { @Override @@ -32,6 +36,7 @@ public String name() { }; } + @EntitlementTest(expectedAccess = PLUGINS) static void httpClientSend() throws InterruptedException { HttpClient httpClient = HttpClient.newBuilder().build(); try { @@ -41,6 +46,7 @@ static void httpClientSend() throws InterruptedException { } } + @EntitlementTest(expectedAccess = PLUGINS) static void httpClientSendAsync() { HttpClient httpClient = HttpClient.newBuilder().build(); httpClient.sendAsync(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding()); diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main19/java/org/elasticsearch/entitlement/qa/test/NativeActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main19/java/org/elasticsearch/entitlement/qa/test/NativeActions.java new file mode 100644 index 0000000000000..f4ed48df3a3dd --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main19/java/org/elasticsearch/entitlement/qa/test/NativeActions.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.MemorySession; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static java.lang.foreign.ValueLayout.ADDRESS; +import static java.lang.foreign.ValueLayout.JAVA_LONG; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +class NativeActions { + + @EntitlementTest(expectedAccess = PLUGINS) + static void linkerDowncallHandle() { + Linker linker = Linker.nativeLinker(); + linker.downcallHandle(FunctionDescriptor.of(JAVA_LONG, ADDRESS)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void linkerDowncallHandleWithAddress() { + Linker linker = Linker.nativeLinker(); + linker.downcallHandle(linker.defaultLookup().lookup("strlen").get(), FunctionDescriptor.of(JAVA_LONG, ADDRESS)); + } + + static int callback() { + return 0; + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void linkerUpcallStub() throws NoSuchMethodException { + Linker linker = Linker.nativeLinker(); + + MethodHandle mh = null; + try { + mh = MethodHandles.lookup().findStatic(NativeActions.class, "callback", MethodType.methodType(int.class)); + } catch (IllegalAccessException e) { + assert false; + } + + FunctionDescriptor callbackDescriptor = FunctionDescriptor.of(ValueLayout.JAVA_INT); + linker.upcallStub(mh, callbackDescriptor, MemorySession.openImplicit()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpret() { + MemorySession scope = MemorySession.openImplicit(); + MemorySegment someSegment; + try { + someSegment = MemorySegment.allocateNative(100, scope); + var foreign = someSegment.get(ValueLayout.ADDRESS, 0); + var segment = MemorySegment.ofAddress(foreign, 4, scope); + } finally { + someSegment = null; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void symbolLookupWithPath() { + try { + SymbolLookup.libraryLookup(FileCheckActions.readDir().resolve("libFoo.so"), MemorySession.openImplicit()); + } catch (IllegalArgumentException e) { + // IllegalArgumentException is thrown if path does not point to a valid library (and it does not) + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void symbolLookupWithName() { + try { + SymbolLookup.libraryLookup("foo", MemorySession.openImplicit()); + } catch (IllegalArgumentException e) { + // IllegalArgumentException is thrown if path does not point to a valid library (and it does not) + } + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main19/java/org/elasticsearch/entitlement/qa/test/VersionSpecificManageThreadsActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main19/java/org/elasticsearch/entitlement/qa/test/VersionSpecificManageThreadsActions.java new file mode 100644 index 0000000000000..fef7d2fcddabf --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main19/java/org/elasticsearch/entitlement/qa/test/VersionSpecificManageThreadsActions.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.core.SuppressForbidden; + +import java.util.concurrent.ForkJoinPool; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +@SuppressForbidden(reason = "testing entitlements") +@SuppressWarnings("unused") // used via reflection +class VersionSpecificManageThreadsActions { + private VersionSpecificManageThreadsActions() {} + + @EntitlementTest(expectedAccess = PLUGINS) + static void java_util_concurrent_ForkJoinPool$setParallelism() { + ForkJoinPool.commonPool().setParallelism(ForkJoinPool.commonPool().getParallelism()); + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main20/java/org/elasticsearch/entitlement/qa/test/NativeActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main20/java/org/elasticsearch/entitlement/qa/test/NativeActions.java new file mode 100644 index 0000000000000..71f9154251e0b --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main20/java/org/elasticsearch/entitlement/qa/test/NativeActions.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import org.elasticsearch.entitlement.qa.entitled.EntitledPlugin; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentScope; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.module.Configuration; +import java.lang.module.ModuleFinder; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; + +import static java.lang.foreign.ValueLayout.ADDRESS; +import static java.lang.foreign.ValueLayout.JAVA_LONG; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; + +class NativeActions { + + @EntitlementTest(expectedAccess = SERVER_ONLY) + static void enableNativeAccess() throws Exception { + ModuleLayer parent = ModuleLayer.boot(); + + var location = EntitledPlugin.class.getProtectionDomain().getCodeSource().getLocation(); + + // We create a layer for our own module, so we have a controller to try and call enableNativeAccess on it. + // This works in both the modular and non-modular case: the target module has to be present in the new layer, but its entitlements + // and policies do not matter to us: we are checking that the caller is (or isn't) entitled to use enableNativeAccess + Configuration cf = parent.configuration() + .resolve(ModuleFinder.of(Path.of(location.toURI())), ModuleFinder.of(), Set.of("org.elasticsearch.entitlement.qa.entitled")); + var controller = ModuleLayer.defineModulesWithOneLoader(cf, List.of(parent), ClassLoader.getSystemClassLoader()); + var targetModule = controller.layer().findModule("org.elasticsearch.entitlement.qa.entitled"); + + controller.enableNativeAccess(targetModule.get()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void linkerDowncallHandle() { + Linker linker = Linker.nativeLinker(); + linker.downcallHandle(FunctionDescriptor.of(JAVA_LONG, ADDRESS)); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void linkerDowncallHandleWithAddress() { + Linker linker = Linker.nativeLinker(); + linker.downcallHandle(linker.defaultLookup().find("strlen").get(), FunctionDescriptor.of(JAVA_LONG, ADDRESS)); + } + + static int callback() { + return 0; + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void linkerUpcallStub() throws NoSuchMethodException { + Linker linker = Linker.nativeLinker(); + + MethodHandle mh = null; + try { + mh = MethodHandles.lookup().findStatic(NativeActions.class, "callback", MethodType.methodType(int.class)); + } catch (IllegalAccessException e) { + assert false; + } + + FunctionDescriptor callbackDescriptor = FunctionDescriptor.of(ValueLayout.JAVA_INT); + linker.upcallStub(mh, callbackDescriptor, SegmentScope.auto()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpretAsUnbounded() { + SegmentScope scope = SegmentScope.global(); + MemorySegment someSegment; + try { + someSegment = MemorySegment.allocateNative(100, scope); + MemorySegment foreign = someSegment.get(ValueLayout.ADDRESS.asUnbounded(), 0); // wrap address into segment (size = + // Long.MAX_VALUE) + } finally { + someSegment = null; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpret() { + SegmentScope scope = SegmentScope.global(); + MemorySegment someSegment; + try { + someSegment = MemorySegment.allocateNative(100, scope); + MemorySegment foreign = someSegment.get(ValueLayout.ADDRESS, 0); + MemorySegment segment = MemorySegment.ofAddress(foreign.address()); + } finally { + someSegment = null; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpretWithoutScope() { + SegmentScope scope = SegmentScope.global(); + MemorySegment someSegment; + try { + someSegment = MemorySegment.allocateNative(100, scope); + MemorySegment foreign = someSegment.get(ValueLayout.ADDRESS, 0); + MemorySegment segment = MemorySegment.ofAddress(foreign.address(), 4); + } finally { + someSegment = null; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpretWithCleanup() { + SegmentScope scope = SegmentScope.global(); + MemorySegment someSegment; + try { + someSegment = MemorySegment.allocateNative(100, scope); + MemorySegment foreign = someSegment.get(ValueLayout.ADDRESS, 0); + MemorySegment segment = MemorySegment.ofAddress(foreign.address(), 4, scope, () -> {}); + } finally { + someSegment = null; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpretWithSize() { + SegmentScope scope = SegmentScope.global(); + MemorySegment someSegment; + try { + someSegment = MemorySegment.allocateNative(100, scope); + MemorySegment foreign = someSegment.get(ValueLayout.ADDRESS, 0); + MemorySegment segment = MemorySegment.ofAddress(foreign.address(), 4, scope); + } finally { + someSegment = null; + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void symbolLookupWithPath() { + try { + SymbolLookup.libraryLookup(FileCheckActions.readDir().resolve("libFoo.so"), SegmentScope.auto()); + } catch (IllegalArgumentException e) { + // IllegalArgumentException is thrown if path does not point to a valid library (and it does not) + } + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void symbolLookupWithName() { + try { + SymbolLookup.libraryLookup("foo", SegmentScope.auto()); + } catch (IllegalArgumentException e) { + // IllegalArgumentException is thrown if path does not point to a valid library (and it does not) + } + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main20/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNioFileSystemActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main20/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNioFileSystemActions.java new file mode 100644 index 0000000000000..ccc3b548632b8 --- /dev/null +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main20/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNioFileSystemActions.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.test; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.attribute.BasicFileAttributes; + +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; + +class VersionSpecificNioFileSystemActions { + @EntitlementTest(expectedAccess = PLUGINS) + static void checkReadAttributesIfExists() throws IOException { + var fs = FileSystems.getDefault().provider(); + fs.readAttributesIfExists(FileCheckActions.readFile(), BasicFileAttributes.class); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void checkExists() { + var fs = FileSystems.getDefault().provider(); + fs.exists(FileCheckActions.readFile()); + } +} diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main22/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNativeChecks.java b/libs/entitlement/qa/entitlement-test-plugin/src/main21/java/org/elasticsearch/entitlement/qa/test/NativeActions.java similarity index 82% rename from libs/entitlement/qa/entitlement-test-plugin/src/main22/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNativeChecks.java rename to libs/entitlement/qa/entitlement-test-plugin/src/main21/java/org/elasticsearch/entitlement/qa/test/NativeActions.java index 0a69f7255a200..ba07de5632ff4 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main22/java/org/elasticsearch/entitlement/qa/test/VersionSpecificNativeChecks.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main21/java/org/elasticsearch/entitlement/qa/test/NativeActions.java @@ -30,9 +30,12 @@ import static java.lang.foreign.ValueLayout.ADDRESS; import static java.lang.foreign.ValueLayout.JAVA_LONG; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; +import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY; -class VersionSpecificNativeChecks { +class NativeActions { + @EntitlementTest(expectedAccess = SERVER_ONLY) static void enableNativeAccess() throws Exception { ModuleLayer parent = ModuleLayer.boot(); @@ -49,16 +52,19 @@ static void enableNativeAccess() throws Exception { controller.enableNativeAccess(targetModule.get()); } + @EntitlementTest(expectedAccess = PLUGINS) static void addressLayoutWithTargetLayout() { AddressLayout addressLayout = ADDRESS.withoutTargetLayout(); addressLayout.withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, ValueLayout.JAVA_BYTE)); } + @EntitlementTest(expectedAccess = PLUGINS) static void linkerDowncallHandle() { Linker linker = Linker.nativeLinker(); linker.downcallHandle(FunctionDescriptor.of(JAVA_LONG, ADDRESS)); } + @EntitlementTest(expectedAccess = PLUGINS) static void linkerDowncallHandleWithAddress() { Linker linker = Linker.nativeLinker(); linker.downcallHandle(linker.defaultLookup().find("strlen").get(), FunctionDescriptor.of(JAVA_LONG, ADDRESS)); @@ -68,12 +74,13 @@ static int callback() { return 0; } + @EntitlementTest(expectedAccess = PLUGINS) static void linkerUpcallStub() throws NoSuchMethodException { Linker linker = Linker.nativeLinker(); MethodHandle mh = null; try { - mh = MethodHandles.lookup().findStatic(VersionSpecificNativeChecks.class, "callback", MethodType.methodType(int.class)); + mh = MethodHandles.lookup().findStatic(NativeActions.class, "callback", MethodType.methodType(int.class)); } catch (IllegalAccessException e) { assert false; } @@ -82,32 +89,37 @@ static void linkerUpcallStub() throws NoSuchMethodException { linker.upcallStub(mh, callbackDescriptor, Arena.ofAuto()); } + @EntitlementTest(expectedAccess = PLUGINS) static void memorySegmentReinterpret() { Arena arena = Arena.ofAuto(); MemorySegment segment = arena.allocate(100); segment.reinterpret(50); } + @EntitlementTest(expectedAccess = PLUGINS) static void memorySegmentReinterpretWithCleanup() { Arena arena = Arena.ofAuto(); MemorySegment segment = arena.allocate(100); segment.reinterpret(Arena.ofAuto(), s -> {}); } - static void memorySegmentReinterpretWithSizeAndCleanup() { + @EntitlementTest(expectedAccess = PLUGINS) + static void memorySegmentReinterpretWithSize() { Arena arena = Arena.ofAuto(); MemorySegment segment = arena.allocate(100); segment.reinterpret(50, Arena.ofAuto(), s -> {}); } + @EntitlementTest(expectedAccess = PLUGINS) static void symbolLookupWithPath() { try { - SymbolLookup.libraryLookup(Path.of("/foo/bar/libFoo.so"), Arena.ofAuto()); + SymbolLookup.libraryLookup(FileCheckActions.readDir().resolve("libFoo.so"), Arena.ofAuto()); } catch (IllegalArgumentException e) { // IllegalArgumentException is thrown if path does not point to a valid library (and it does not) } } + @EntitlementTest(expectedAccess = PLUGINS) static void symbolLookupWithName() { try { SymbolLookup.libraryLookup("foo", Arena.ofAuto()); diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java index 487f692ef4488..d24fd32ade6ae 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java @@ -11,34 +11,45 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.entitlement.qa.EntitlementsTestRule.PolicyBuilder; import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; import java.io.IOException; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public abstract class AbstractEntitlementsIT extends ESRestTestCase { - static final EntitlementsTestRule.PolicyBuilder ALLOWED_TEST_ENTITLEMENTS = (builder, tempDir) -> { + static final PolicyBuilder ALLOWED_TEST_ENTITLEMENTS = (builder, tempDir) -> { builder.value("create_class_loader"); builder.value("set_https_connection_properties"); builder.value("inbound_network"); builder.value("outbound_network"); builder.value("load_native_libraries"); + builder.value("manage_threads"); builder.value( Map.of( "write_system_properties", Map.of("properties", List.of("es.entitlements.checkSetSystemProperty", "es.entitlements.checkClearSystemProperty")) ) ); - - builder.value(Map.of("file", Map.of("path", tempDir.resolve("read_dir"), "mode", "read"))); - builder.value(Map.of("file", Map.of("path", tempDir.resolve("read_write_dir"), "mode", "read_write"))); - builder.value(Map.of("file", Map.of("path", tempDir.resolve("read_file"), "mode", "read"))); - builder.value(Map.of("file", Map.of("path", tempDir.resolve("read_write_file"), "mode", "read_write"))); + builder.value( + Map.of( + "files", + List.of( + Map.of("path", tempDir.resolve("read_dir"), "mode", "read"), + Map.of("path", tempDir.resolve("read_write_dir"), "mode", "read_write"), + Map.of("path", tempDir.resolve("read_file"), "mode", "read"), + Map.of("path", tempDir.resolve("read_write_file"), "mode", "read_write") + ) + ) + ); }; private final String actionName; @@ -61,8 +72,34 @@ public void testAction() throws IOException { Response result = executeCheck(); assertThat(result.getStatusLine().getStatusCode(), equalTo(200)); } else { - var exception = expectThrows(IOException.class, this::executeCheck); - assertThat(exception.getMessage(), containsString("not_entitled_exception")); + var exception = expectThrows(ResponseException.class, this::executeCheck); + assertThat(exception, statusCodeMatcher(403)); } } + + private static Matcher statusCodeMatcher(int statusCode) { + return new TypeSafeMatcher<>() { + String expectedException = null; + + @Override + protected boolean matchesSafely(ResponseException item) { + Response resp = item.getResponse(); + expectedException = resp.getHeader("expectedException"); + return resp.getStatusLine().getStatusCode() == statusCode && expectedException != null; + } + + @Override + public void describeTo(Description description) { + description.appendValue(statusCode).appendText(" due to ").appendText(expectedException); + } + + @Override + protected void describeMismatchSafely(ResponseException item, Description description) { + description.appendText("was ") + .appendValue(item.getResponse().getStatusLine().getStatusCode()) + .appendText("\n") + .appendValue(item.getMessage()); + } + }; + } } diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedViaOverrideIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedViaOverrideIT.java new file mode 100644 index 0000000000000..37e9a9153922a --- /dev/null +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedViaOverrideIT.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.core.Strings; +import org.junit.ClassRule; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Map; +import java.util.stream.Stream; + +import static org.elasticsearch.entitlement.qa.EntitlementsTestRule.ENTITLEMENT_QA_TEST_MODULE_NAME; +import static org.elasticsearch.entitlement.qa.EntitlementsTestRule.ENTITLEMENT_TEST_PLUGIN_NAME; + +public class EntitlementsAllowedViaOverrideIT extends AbstractEntitlementsIT { + + private static Map createPolicyOverrideSystemProperty(Path tempDir) { + String policyOverride = Strings.format(""" + policy: + %s: + - load_native_libraries + - files: + - path: %s + mode: read + """, ENTITLEMENT_QA_TEST_MODULE_NAME, tempDir.resolve("read_dir")); + var encodedPolicyOverride = new String(Base64.getEncoder().encode(policyOverride.getBytes(StandardCharsets.UTF_8))); + return Map.of("es.entitlements.policy." + ENTITLEMENT_TEST_PLUGIN_NAME, encodedPolicyOverride); + } + + @ClassRule + public static EntitlementsTestRule testRule = new EntitlementsTestRule( + true, + null, + EntitlementsAllowedViaOverrideIT::createPolicyOverrideSystemProperty + ); + + public EntitlementsAllowedViaOverrideIT(@Name("actionName") String actionName) { + super(actionName, true); + } + + @ParametersFactory + public static Iterable data() { + return Stream.of("runtimeLoadLibrary", "fileList").map(action -> new Object[] { action }).toList(); + } + + @Override + protected String getTestRestCluster() { + return testRule.cluster.getHttpAddresses(); + } +} diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAlwaysAllowedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAlwaysAllowedIT.java new file mode 100644 index 0000000000000..36e5b6dd4b8ac --- /dev/null +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAlwaysAllowedIT.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction; +import org.junit.ClassRule; + +public class EntitlementsAlwaysAllowedIT extends AbstractEntitlementsIT { + + @ClassRule + public static EntitlementsTestRule testRule = new EntitlementsTestRule(true, null); + + public EntitlementsAlwaysAllowedIT(@Name("actionName") String actionName) { + super(actionName, true); + } + + @ParametersFactory + public static Iterable data() { + return RestEntitlementsCheckAction.getAlwaysAllowedCheckActions().stream().map(action -> new Object[] { action }).toList(); + } + + @Override + protected String getTestRestCluster() { + return testRule.cluster.getHttpAddresses(); + } +} diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAlwaysAllowedNonModularIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAlwaysAllowedNonModularIT.java new file mode 100644 index 0000000000000..42c2732da34a7 --- /dev/null +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAlwaysAllowedNonModularIT.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction; +import org.junit.ClassRule; + +public class EntitlementsAlwaysAllowedNonModularIT extends AbstractEntitlementsIT { + + @ClassRule + public static EntitlementsTestRule testRule = new EntitlementsTestRule(false, null); + + public EntitlementsAlwaysAllowedNonModularIT(@Name("actionName") String actionName) { + super(actionName, true); + } + + @ParametersFactory + public static Iterable data() { + return RestEntitlementsCheckAction.getAlwaysAllowedCheckActions().stream().map(action -> new Object[] { action }).toList(); + } + + @Override + protected String getTestRestCluster() { + return testRule.cluster.getHttpAddresses(); + } +} diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java index 6f348d38d8e53..5d31afbd8a5b3 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java @@ -26,7 +26,7 @@ public EntitlementsDeniedIT(@Name("actionName") String actionName) { @ParametersFactory public static Iterable data() { - return RestEntitlementsCheckAction.getAllCheckActions().stream().map(action -> new Object[] { action }).toList(); + return RestEntitlementsCheckAction.getDeniableCheckActions().stream().map(action -> new Object[] { action }).toList(); } @Override diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedNonModularIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedNonModularIT.java index 6f2003f7275d4..ece18d4830387 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedNonModularIT.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedNonModularIT.java @@ -26,7 +26,7 @@ public EntitlementsDeniedNonModularIT(@Name("actionName") String actionName) { @ParametersFactory public static Iterable data() { - return RestEntitlementsCheckAction.getAllCheckActions().stream().map(action -> new Object[] { action }).toList(); + return RestEntitlementsCheckAction.getDeniableCheckActions().stream().map(action -> new Object[] { action }).toList(); } @Override diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java index 33d5eeca595ab..9cad8b710ae11 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java @@ -26,19 +26,49 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; class EntitlementsTestRule implements TestRule { + // entitlements that test methods may use, see EntitledActions + private static final PolicyBuilder ENTITLED_POLICY = (builder, tempDir) -> { + builder.value("manage_threads"); + builder.value("outbound_network"); + builder.value( + Map.of( + "files", + List.of( + Map.of("path", tempDir.resolve("read_dir"), "mode", "read_write"), + Map.of("path", tempDir.resolve("read_dir").resolve("k8s").resolve("..data"), "mode", "read", "exclusive", true), + Map.of("path", tempDir.resolve("read_write_dir"), "mode", "read_write"), + Map.of("path", tempDir.resolve("read_file"), "mode", "read"), + Map.of("path", tempDir.resolve("read_write_file"), "mode", "read_write") + ) + ) + ); + }; + public static final String ENTITLEMENT_QA_TEST_MODULE_NAME = "org.elasticsearch.entitlement.qa.test"; + public static final String ENTITLEMENT_TEST_PLUGIN_NAME = "entitlement-test-plugin"; + interface PolicyBuilder { void build(XContentBuilder builder, Path tempDir) throws IOException; } + interface TempDirSystemPropertyProvider { + Map get(Path tempDir); + } + final TemporaryFolder testDir; final ElasticsearchCluster cluster; final TestRule ruleChain; - @SuppressWarnings("this-escape") EntitlementsTestRule(boolean modular, PolicyBuilder policyBuilder) { + this(modular, policyBuilder, tempDir -> Map.of()); + } + + @SuppressWarnings("this-escape") + EntitlementsTestRule(boolean modular, PolicyBuilder policyBuilder, TempDirSystemPropertyProvider tempDirSystemPropertyProvider) { testDir = new TemporaryFolder(); var tempDirSetup = new ExternalResource() { @Override @@ -51,11 +81,15 @@ protected void before() throws Throwable { } }; cluster = ElasticsearchCluster.local() - .module("entitled") - .module("entitlement-test-plugin", spec -> setupEntitlements(spec, modular, policyBuilder)) + .module("entitled", spec -> buildEntitlements(spec, "org.elasticsearch.entitlement.qa.entitled", ENTITLED_POLICY)) + .module(ENTITLEMENT_TEST_PLUGIN_NAME, spec -> setupEntitlements(spec, modular, policyBuilder)) .systemProperty("es.entitlements.enabled", "true") + .systemProperty("es.entitlements.verify_bytecode", "true") .systemProperty("es.entitlements.testdir", () -> testDir.getRoot().getAbsolutePath()) + .systemProperties(spec -> tempDirSystemPropertyProvider.get(testDir.getRoot().toPath())) .setting("xpack.security.enabled", "false") + // Logs in libs/entitlement/qa/build/test-results/javaRestTest/TEST-org.elasticsearch.entitlement.qa.EntitlementsXXX.xml + // .setting("logger.org.elasticsearch.entitlement", "DEBUG") .build(); ruleChain = RuleChain.outerRule(testDir).around(tempDirSetup).around(cluster); } @@ -65,34 +99,35 @@ public Statement apply(Statement statement, Description description) { return ruleChain.apply(statement, description); } + private void buildEntitlements(PluginInstallSpec spec, String moduleName, PolicyBuilder policyBuilder) { + spec.withEntitlementsOverride(old -> { + try (var builder = YamlXContent.contentBuilder()) { + builder.startObject(); + builder.field(moduleName); + builder.startArray(); + + policyBuilder.build(builder, testDir.getRoot().toPath()); + builder.endArray(); + builder.endObject(); + + String policy = Strings.toString(builder); + System.out.println("Using entitlement policy for module " + moduleName + ":\n" + policy); + return Resource.fromString(policy); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + private void setupEntitlements(PluginInstallSpec spec, boolean modular, PolicyBuilder policyBuilder) { - String moduleName = modular ? "org.elasticsearch.entitlement.qa.test" : "ALL-UNNAMED"; + String moduleName = modular ? ENTITLEMENT_QA_TEST_MODULE_NAME : "ALL-UNNAMED"; if (policyBuilder != null) { - spec.withEntitlementsOverride(old -> { - try { - try (var builder = YamlXContent.contentBuilder()) { - builder.startObject(); - builder.field(moduleName); - builder.startArray(); - - policyBuilder.build(builder, testDir.getRoot().toPath()); - builder.endArray(); - builder.endObject(); - - String policy = Strings.toString(builder); - System.out.println("Using entitlement policy:\n" + policy); - return Resource.fromString(policy); - } - - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + buildEntitlements(spec, moduleName, policyBuilder); } if (modular == false) { spec.withPropertiesOverride(old -> { - String props = old.replace("modulename=org.elasticsearch.entitlement.qa.test", ""); + String props = old.replace("modulename=" + ENTITLEMENT_QA_TEST_MODULE_NAME, ""); System.out.println("Using plugin properties:\n" + props); return Resource.fromString(props); }); diff --git a/libs/entitlement/src/main/java/module-info.java b/libs/entitlement/src/main/java/module-info.java index c0959f212558a..d6737a14a0b88 100644 --- a/libs/entitlement/src/main/java/module-info.java +++ b/libs/entitlement/src/main/java/module-info.java @@ -8,17 +8,20 @@ */ module org.elasticsearch.entitlement { + requires org.elasticsearch.base; requires org.elasticsearch.xcontent; requires org.elasticsearch.logging; requires java.instrument; - requires org.elasticsearch.base; - requires jdk.attach; + requires java.logging; requires java.net.http; + requires jdk.attach; + requires jdk.net; requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base exports org.elasticsearch.entitlement.runtime.api; exports org.elasticsearch.entitlement.runtime.policy; + exports org.elasticsearch.entitlement.runtime.policy.entitlements to org.elasticsearch.server; exports org.elasticsearch.entitlement.instrumentation; exports org.elasticsearch.entitlement.bootstrap to org.elasticsearch.server; exports org.elasticsearch.entitlement.initialization to java.base; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 496a28a448381..a2c6307185b29 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -14,9 +14,9 @@ import com.sun.tools.attach.AttachNotSupportedException; import com.sun.tools.attach.VirtualMachine; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.initialization.EntitlementInitialization; -import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -25,16 +25,48 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Stream; import static java.util.Objects.requireNonNull; public class EntitlementBootstrap { - public record BootstrapArgs(Map pluginPolicies, Function, String> pluginResolver) { + public record BootstrapArgs( + @Nullable Policy serverPolicyPatch, + Map pluginPolicies, + Function, String> pluginResolver, + Function> settingResolver, + Path[] dataDirs, + Path[] sharedRepoDirs, + Path configDir, + Path libDir, + Path modulesDir, + Path pluginsDir, + Map sourcePaths, + Path logsDir, + Path tempDir, + Path pidFile, + Set> suppressFailureLogClasses + ) { public BootstrapArgs { requireNonNull(pluginPolicies); requireNonNull(pluginResolver); + requireNonNull(settingResolver); + requireNonNull(dataDirs); + if (dataDirs.length == 0) { + throw new IllegalArgumentException("must provide at least one data directory"); + } + requireNonNull(sharedRepoDirs); + requireNonNull(configDir); + requireNonNull(libDir); + requireNonNull(modulesDir); + requireNonNull(pluginsDir); + requireNonNull(sourcePaths); + requireNonNull(logsDir); + requireNonNull(tempDir); + requireNonNull(suppressFailureLogClasses); } } @@ -48,18 +80,62 @@ public static BootstrapArgs bootstrapArgs() { * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}. * + * @param serverPolicyPatch a policy with additional entitlements to patch the embedded server layer policy * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name. * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name). + * @param settingResolver a functor to resolve a setting name pattern for one or more Elasticsearch settings. + * @param dataDirs data directories for Elasticsearch + * @param sharedRepoDirs shared repository directories for Elasticsearch + * @param configDir the config directory for Elasticsearch + * @param libDir the lib directory for Elasticsearch + * @param modulesDir the directory where Elasticsearch modules are + * @param pluginsDir the directory where plugins are installed for Elasticsearch + * @param sourcePaths a map holding the path to each plugin or module jars, by plugin (or module) name. + * @param tempDir the temp directory for Elasticsearch + * @param logsDir the log directory for Elasticsearch + * @param pidFile path to a pid file for Elasticsearch, or {@code null} if one was not specified + * @param suppressFailureLogClasses classes for which we do not need or want to log Entitlements failures */ - public static void bootstrap(Map pluginPolicies, Function, String> pluginResolver) { + public static void bootstrap( + Policy serverPolicyPatch, + Map pluginPolicies, + Function, String> pluginResolver, + Function> settingResolver, + Path[] dataDirs, + Path[] sharedRepoDirs, + Path configDir, + Path libDir, + Path modulesDir, + Path pluginsDir, + Map sourcePaths, + Path logsDir, + Path tempDir, + Path pidFile, + Set> suppressFailureLogClasses + ) { logger.debug("Loading entitlement agent"); if (EntitlementBootstrap.bootstrapArgs != null) { throw new IllegalStateException("plugin data is already set"); } - EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver); + EntitlementBootstrap.bootstrapArgs = new BootstrapArgs( + serverPolicyPatch, + pluginPolicies, + pluginResolver, + settingResolver, + dataDirs, + sharedRepoDirs, + configDir, + libDir, + modulesDir, + pluginsDir, + sourcePaths, + logsDir, + tempDir, + pidFile, + suppressFailureLogClasses + ); exportInitializationToAgent(); loadAgent(findAgentJar()); - selfTest(); } @SuppressForbidden(reason = "The VirtualMachine API is the only way to attach a java agent dynamically") @@ -105,63 +181,5 @@ private static String findAgentJar() { } } - /** - * Attempt a few sensitive operations to ensure that some are permitted and some are forbidden. - *

- * - * This serves two purposes: - * - *

    - *
  1. - * a smoke test to make sure the entitlements system is not completely broken, and - *
    2. - *
  2. - * an early test of certain important operations so they don't fail later on at an awkward time. - *
    3. - *
- * - * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing - */ - private static void selfTest() { - ensureCannotStartProcess(); - ensureCanCreateTempFile(); - } - - private static void ensureCannotStartProcess() { - try { - // The command doesn't matter; it doesn't even need to exist - new ProcessBuilder("").start(); - } catch (NotEntitledException e) { - logger.debug("Success: Entitlement protection correctly prevented process creation"); - return; - } catch (IOException e) { - throw new IllegalStateException("Failed entitlement protection self-test", e); - } - throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); - } - - /** - * Originally {@code Security.selfTest}. - */ - @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test") - private static void ensureCanCreateTempFile() { - try { - Path p = Files.createTempFile(null, null); - p.toFile().deleteOnExit(); - - // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally. - try { - Files.delete(p); - } catch (IOException ignored) { - // Can be caused by virus scanner - } - } catch (NotEntitledException e) { - throw new IllegalStateException("Entitlement protection self-test was incorrectly forbidden", e); - } catch (Exception e) { - throw new IllegalStateException("Unable to perform entitlement protection self-test", e); - } - logger.debug("Success: Entitlement protection correctly permitted temp file creation"); - } - private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index ead3d5c882b19..ff06da09ed69c 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -9,6 +9,8 @@ package org.elasticsearch.entitlement.initialization; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.internal.provider.ProviderLocator; import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; import org.elasticsearch.entitlement.bridge.EntitlementChecker; @@ -18,26 +20,59 @@ import org.elasticsearch.entitlement.instrumentation.MethodKey; import org.elasticsearch.entitlement.instrumentation.Transformer; import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker; -import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement; -import org.elasticsearch.entitlement.runtime.policy.Entitlement; -import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement; -import org.elasticsearch.entitlement.runtime.policy.InboundNetworkEntitlement; -import org.elasticsearch.entitlement.runtime.policy.LoadNativeLibrariesEntitlement; -import org.elasticsearch.entitlement.runtime.policy.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.PathLookup; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; +import org.elasticsearch.entitlement.runtime.policy.PolicyUtils; import org.elasticsearch.entitlement.runtime.policy.Scope; +import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData; +import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; import java.lang.instrument.Instrumentation; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.nio.channels.spi.SelectorProvider; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileSystems; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchService; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static org.elasticsearch.entitlement.runtime.policy.Platform.LINUX; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.CONFIG; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; /** * Called by the agent during {@code agentmain} to configure the entitlement system, @@ -53,6 +88,11 @@ public class EntitlementInitialization { private static ElasticsearchEntitlementChecker manager; + interface InstrumentationInfoFactory { + InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) throws ClassNotFoundException, + NoSuchMethodException; + } + // Note: referenced by bridge reflectively public static EntitlementChecker checker() { return manager; @@ -62,26 +102,51 @@ public static EntitlementChecker checker() { public static void initialize(Instrumentation inst) throws Exception { manager = initChecker(); - Map checkMethods = new HashMap<>(); - int javaVersion = Runtime.version().feature(); - Set> interfaces = new HashSet<>(); - for (int i = 17; i <= javaVersion; ++i) { - interfaces.add(getVersionSpecificCheckerClass(i, "org.elasticsearch.entitlement.bridge", "EntitlementChecker")); - } - for (var checkerInterface : interfaces) { - checkMethods.putAll(INSTRUMENTER_FACTORY.lookupMethods(checkerInterface)); + var latestCheckerInterface = getVersionSpecificCheckerClass(EntitlementChecker.class, Runtime.version().feature()); + var verifyBytecode = Booleans.parseBoolean(System.getProperty("es.entitlements.verify_bytecode", "false")); + + if (verifyBytecode) { + ensureClassesSensitiveToVerificationAreInitialized(); } - var latestCheckerInterface = getVersionSpecificCheckerClass( - javaVersion, - "org.elasticsearch.entitlement.bridge", - "EntitlementChecker" - ); + Map checkMethods = new HashMap<>(INSTRUMENTATION_SERVICE.lookupMethods(latestCheckerInterface)); + Stream.of( + fileSystemProviderChecks(), + fileStoreChecks(), + pathChecks(), + Stream.of( + INSTRUMENTATION_SERVICE.lookupImplementationMethod( + SelectorProvider.class, + "inheritedChannel", + SelectorProvider.provider().getClass(), + EntitlementChecker.class, + "checkSelectorProviderInheritedChannel" + ) + ) + ) + .flatMap(Function.identity()) + .forEach(instrumentation -> checkMethods.put(instrumentation.targetMethod(), instrumentation.checkMethod())); + var classesToTransform = checkMethods.keySet().stream().map(MethodKey::className).collect(Collectors.toSet()); - Instrumenter instrumenter = INSTRUMENTER_FACTORY.newInstrumenter(latestCheckerInterface, checkMethods); - inst.addTransformer(new Transformer(instrumenter, classesToTransform), true); - inst.retransformClasses(findClassesToRetransform(inst.getAllLoadedClasses(), classesToTransform)); + Instrumenter instrumenter = INSTRUMENTATION_SERVICE.newInstrumenter(latestCheckerInterface, checkMethods); + var transformer = new Transformer(instrumenter, classesToTransform, verifyBytecode); + inst.addTransformer(transformer, true); + + var classesToRetransform = findClassesToRetransform(inst.getAllLoadedClasses(), classesToTransform); + try { + inst.retransformClasses(classesToRetransform); + } catch (VerifyError e) { + // Turn on verification and try to retransform one class at the time to get detailed diagnostic + transformer.enableClassVerification(); + + for (var classToRetransform : classesToRetransform) { + inst.retransformClasses(classToRetransform); + } + + // We should have failed already in the loop above, but just in case we did not, rethrow. + throw e; + } } private static Class[] findClassesToRetransform(Class[] loadedClasses, Set classesToTransform) { @@ -95,69 +160,368 @@ private static Class[] findClassesToRetransform(Class[] loadedClasses, Set } private static PolicyManager createPolicyManager() { - Map pluginPolicies = EntitlementBootstrap.bootstrapArgs().pluginPolicies(); + EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs(); + Map pluginPolicies = bootstrapArgs.pluginPolicies(); + var pathLookup = new PathLookup( + getUserHome(), + bootstrapArgs.configDir(), + bootstrapArgs.dataDirs(), + bootstrapArgs.sharedRepoDirs(), + bootstrapArgs.tempDir(), + bootstrapArgs.settingResolver() + ); - // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it - var serverPolicy = new Policy( - "server", - List.of( - new Scope("org.elasticsearch.base", List.of(new CreateClassLoaderEntitlement())), - new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())), + List serverScopes = new ArrayList<>(); + List serverModuleFileDatas = new ArrayList<>(); + Collections.addAll( + serverModuleFileDatas, + // Base ES directories + FileData.ofPath(bootstrapArgs.pluginsDir(), READ), + FileData.ofPath(bootstrapArgs.modulesDir(), READ), + FileData.ofPath(bootstrapArgs.configDir(), READ), + FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE), + FileData.ofPath(bootstrapArgs.libDir(), READ), + FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE), + FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE), + // exclusive settings file + FileData.ofRelativePath(Path.of("operator/settings.json"), CONFIG, READ_WRITE).withExclusive(true), + + // OS release on Linux + FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/etc/system-release"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/usr/lib/os-release"), READ).withPlatform(LINUX), + // read max virtual memory areas + FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/proc/meminfo"), READ).withPlatform(LINUX), + // load averages on Linux + FileData.ofPath(Path.of("/proc/loadavg"), READ).withPlatform(LINUX), + // control group stats on Linux. cgroup v2 stats are in an unpredicable + // location under `/sys/fs/cgroup`, so unfortunately we have to allow + // read access to the entire directory hierarchy. + FileData.ofPath(Path.of("/proc/self/cgroup"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ).withPlatform(LINUX), + // // io stats on Linux + FileData.ofPath(Path.of("/proc/self/mountinfo"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/proc/diskstats"), READ).withPlatform(LINUX) + ); + if (bootstrapArgs.pidFile() != null) { + serverModuleFileDatas.add(FileData.ofPath(bootstrapArgs.pidFile(), READ_WRITE)); + } + + Collections.addAll( + serverScopes, + new Scope( + "org.elasticsearch.base", + List.of( + new CreateClassLoaderEntitlement(), + new FilesEntitlement( + List.of( + // TODO: what in es.base is accessing shared repo? + FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE), + FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE) + ) + ) + ) + ), + new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())), + new Scope( + "org.elasticsearch.server", + List.of( + new ExitVMEntitlement(), + new ReadStoreAttributesEntitlement(), + new CreateClassLoaderEntitlement(), + new InboundNetworkEntitlement(), + new OutboundNetworkEntitlement(), + new LoadNativeLibrariesEntitlement(), + new ManageThreadsEntitlement(), + new FilesEntitlement(serverModuleFileDatas) + ) + ), + new Scope("java.desktop", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), + new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())), + new Scope( + "org.apache.lucene.core", + List.of( + new LoadNativeLibrariesEntitlement(), + new ManageThreadsEntitlement(), + new FilesEntitlement( + List.of(FileData.ofPath(bootstrapArgs.configDir(), READ), FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)) + ) + ) + ), + new Scope( + "org.apache.lucene.misc", + List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)))) + ), + new Scope( + "org.apache.logging.log4j.core", + List.of(new ManageThreadsEntitlement(), new FilesEntitlement(List.of(FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE)))) + ), + new Scope( + "org.elasticsearch.nativeaccess", + List.of( + new LoadNativeLibrariesEntitlement(), + new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))) + ) + ) + ); + + // conditionally add FIPS entitlements if FIPS only functionality is enforced + if (Booleans.parseBoolean(System.getProperty("org.bouncycastle.fips.approved_only"), false)) { + // if custom trust store is set, grant read access to its location, otherwise use the default JDK trust store + String trustStore = System.getProperty("javax.net.ssl.trustStore"); + Path trustStorePath = trustStore != null + ? Path.of(trustStore) + : Path.of(System.getProperty("java.home")).resolve("lib/security/jssecacerts"); + + Collections.addAll( + serverScopes, new Scope( - "org.elasticsearch.server", + "org.bouncycastle.fips.tls", List.of( - new ExitVMEntitlement(), - new CreateClassLoaderEntitlement(), - new InboundNetworkEntitlement(), - new OutboundNetworkEntitlement(), - new LoadNativeLibrariesEntitlement() + new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))), + new ManageThreadsEntitlement(), + new OutboundNetworkEntitlement() ) ), - new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), - new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())), - new Scope("org.apache.lucene.core", List.of(new LoadNativeLibrariesEntitlement())), - new Scope("org.elasticsearch.nativeaccess", List.of(new LoadNativeLibrariesEntitlement())) - ) + new Scope( + "org.bouncycastle.fips.core", + // read to lib dir is required for checksum validation + List.of(new FilesEntitlement(List.of(FileData.ofPath(bootstrapArgs.libDir(), READ))), new ManageThreadsEntitlement()) + ) + ); + } + + var serverPolicy = new Policy( + "server", + bootstrapArgs.serverPolicyPatch() == null + ? serverScopes + : PolicyUtils.mergeScopes(serverScopes, bootstrapArgs.serverPolicyPatch().scopes()) ); + // agents run without a module, so this is a special hack for the apm agent // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed - List agentEntitlements = List.of(new CreateClassLoaderEntitlement()); - var resolver = EntitlementBootstrap.bootstrapArgs().pluginResolver(); - return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, AGENTS_PACKAGE_NAME, ENTITLEMENTS_MODULE); + // See also modules/apm/src/main/plugin-metadata/entitlement-policy.yaml + List agentEntitlements = List.of( + new CreateClassLoaderEntitlement(), + new ManageThreadsEntitlement(), + new SetHttpsConnectionPropertiesEntitlement(), + new OutboundNetworkEntitlement(), + new WriteSystemPropertiesEntitlement(Set.of("AsyncProfiler.safemode")), + new LoadNativeLibrariesEntitlement(), + new FilesEntitlement( + List.of( + FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE), + FileData.ofPath(Path.of("/proc/meminfo"), READ), + FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ) + ) + ) + ); + return new PolicyManager( + serverPolicy, + agentEntitlements, + pluginPolicies, + EntitlementBootstrap.bootstrapArgs().pluginResolver(), + EntitlementBootstrap.bootstrapArgs().sourcePaths(), + AGENTS_PACKAGE_NAME, + ENTITLEMENTS_MODULE, + pathLookup, + bootstrapArgs.suppressFailureLogClasses() + ); } - private static ElasticsearchEntitlementChecker initChecker() { - final PolicyManager policyManager = createPolicyManager(); + private static Path getUserHome() { + String userHome = System.getProperty("user.home"); + if (userHome == null) { + throw new IllegalStateException("user.home system property is required"); + } + return PathUtils.get(userHome); + } - Class clazz = getVersionSpecificCheckerClass( - Runtime.version().feature(), - "org.elasticsearch.entitlement.runtime.api", - "ElasticsearchEntitlementChecker" + private static Stream fileSystemProviderChecks() throws ClassNotFoundException, + NoSuchMethodException { + var fileSystemProviderClass = FileSystems.getDefault().provider().getClass(); + + var instrumentation = new InstrumentationInfoFactory() { + @Override + public InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) + throws ClassNotFoundException, NoSuchMethodException { + return INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileSystemProvider.class, + methodName, + fileSystemProviderClass, + EntitlementChecker.class, + "check" + Character.toUpperCase(methodName.charAt(0)) + methodName.substring(1), + parameterTypes + ); + } + }; + + var allVersionsMethods = Stream.of( + instrumentation.of("newFileSystem", URI.class, Map.class), + instrumentation.of("newFileSystem", Path.class, Map.class), + instrumentation.of("newInputStream", Path.class, OpenOption[].class), + instrumentation.of("newOutputStream", Path.class, OpenOption[].class), + instrumentation.of("newFileChannel", Path.class, Set.class, FileAttribute[].class), + instrumentation.of("newAsynchronousFileChannel", Path.class, Set.class, ExecutorService.class, FileAttribute[].class), + instrumentation.of("newByteChannel", Path.class, Set.class, FileAttribute[].class), + instrumentation.of("newDirectoryStream", Path.class, DirectoryStream.Filter.class), + instrumentation.of("createDirectory", Path.class, FileAttribute[].class), + instrumentation.of("createSymbolicLink", Path.class, Path.class, FileAttribute[].class), + instrumentation.of("createLink", Path.class, Path.class), + instrumentation.of("delete", Path.class), + instrumentation.of("deleteIfExists", Path.class), + instrumentation.of("readSymbolicLink", Path.class), + instrumentation.of("copy", Path.class, Path.class, CopyOption[].class), + instrumentation.of("move", Path.class, Path.class, CopyOption[].class), + instrumentation.of("isSameFile", Path.class, Path.class), + instrumentation.of("isHidden", Path.class), + instrumentation.of("getFileStore", Path.class), + instrumentation.of("checkAccess", Path.class, AccessMode[].class), + instrumentation.of("getFileAttributeView", Path.class, Class.class, LinkOption[].class), + instrumentation.of("readAttributes", Path.class, Class.class, LinkOption[].class), + instrumentation.of("readAttributes", Path.class, String.class, LinkOption[].class), + instrumentation.of("setAttribute", Path.class, String.class, Object.class, LinkOption[].class) ); - Constructor constructor; - try { - constructor = clazz.getConstructor(PolicyManager.class); - } catch (NoSuchMethodException e) { - throw new AssertionError("entitlement impl is missing no arg constructor", e); + + if (Runtime.version().feature() >= 20) { + var java20EntitlementCheckerClass = getVersionSpecificCheckerClass(EntitlementChecker.class, 20); + var java20Methods = Stream.of( + INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileSystemProvider.class, + "readAttributesIfExists", + fileSystemProviderClass, + java20EntitlementCheckerClass, + "checkReadAttributesIfExists", + Path.class, + Class.class, + LinkOption[].class + ), + INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileSystemProvider.class, + "exists", + fileSystemProviderClass, + java20EntitlementCheckerClass, + "checkExists", + Path.class, + LinkOption[].class + ) + ); + return Stream.concat(allVersionsMethods, java20Methods); } - try { - return (ElasticsearchEntitlementChecker) constructor.newInstance(policyManager); - } catch (IllegalAccessException | InvocationTargetException | InstantiationException e) { - throw new AssertionError(e); + return allVersionsMethods; + } + + private static Stream fileStoreChecks() { + var fileStoreClasses = StreamSupport.stream(FileSystems.getDefault().getFileStores().spliterator(), false) + .map(FileStore::getClass) + .distinct(); + return fileStoreClasses.flatMap(fileStoreClass -> { + var instrumentation = new InstrumentationInfoFactory() { + @Override + public InstrumentationService.InstrumentationInfo of(String methodName, Class... parameterTypes) + throws ClassNotFoundException, NoSuchMethodException { + return INSTRUMENTATION_SERVICE.lookupImplementationMethod( + FileStore.class, + methodName, + fileStoreClass, + EntitlementChecker.class, + "check" + Character.toUpperCase(methodName.charAt(0)) + methodName.substring(1), + parameterTypes + ); + } + }; + + try { + return Stream.of( + instrumentation.of("getFileStoreAttributeView", Class.class), + instrumentation.of("getAttribute", String.class), + instrumentation.of("getBlockSize"), + instrumentation.of("getTotalSpace"), + instrumentation.of("getUnallocatedSpace"), + instrumentation.of("getUsableSpace"), + instrumentation.of("isReadOnly"), + instrumentation.of("name"), + instrumentation.of("type") + + ); + } catch (NoSuchMethodException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + }); + } + + private static Stream pathChecks() { + var pathClasses = StreamSupport.stream(FileSystems.getDefault().getRootDirectories().spliterator(), false) + .map(Path::getClass) + .distinct(); + return pathClasses.flatMap(pathClass -> { + InstrumentationInfoFactory instrumentation = (String methodName, Class... parameterTypes) -> INSTRUMENTATION_SERVICE + .lookupImplementationMethod( + Path.class, + methodName, + pathClass, + EntitlementChecker.class, + "checkPath" + Character.toUpperCase(methodName.charAt(0)) + methodName.substring(1), + parameterTypes + ); + + try { + return Stream.of( + instrumentation.of("toRealPath", LinkOption[].class), + instrumentation.of("register", WatchService.class, WatchEvent.Kind[].class), + instrumentation.of("register", WatchService.class, WatchEvent.Kind[].class, WatchEvent.Modifier[].class) + ); + } catch (NoSuchMethodException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + }); + } + + /** + * If bytecode verification is enabled, ensure these classes get loaded before transforming/retransforming them. + * For these classes, the order in which we transform and verify them matters. Verification during class transformation is at least an + * unforeseen (if not unsupported) scenario: we are loading a class, and while we are still loading it (during transformation) we try + * to verify it. This in turn leads to more classes loading (for verification purposes), which could turn into those classes to be + * transformed and undergo verification. In order to avoid circularity errors as much as possible, we force a partial order. + */ + private static void ensureClassesSensitiveToVerificationAreInitialized() { + var classesToInitialize = Set.of( + "sun.net.www.protocol.http.HttpURLConnection", + "sun.nio.ch.SocketChannelImpl", + "java.net.ProxySelector" + ); + for (String className : classesToInitialize) { + try { + Class.forName(className); + } catch (ClassNotFoundException unexpected) { + throw new AssertionError(unexpected); + } } } - private static Class getVersionSpecificCheckerClass(int javaVersion, String packageName, String baseClassName) { + /** + * Returns the "most recent" checker class compatible with the current runtime Java version. + * For checkers, we have (optionally) version specific classes, each with a prefix (e.g. Java23). + * The mapping cannot be automatic, as it depends on the actual presence of these classes in the final Jar (see + * the various mainXX source sets). + */ + private static Class getVersionSpecificCheckerClass(Class baseClass, int javaVersion) { + String packageName = baseClass.getPackageName(); + String baseClassName = baseClass.getSimpleName(); + final String classNamePrefix; - if (javaVersion == 21) { - classNamePrefix = "Java21"; - } else if (javaVersion == 22) { - classNamePrefix = "Java22"; - } else if (javaVersion >= 23) { - classNamePrefix = "Java23"; - } else { + if (javaVersion < 19) { + // For older Java versions, the basic EntitlementChecker interface and implementation contains all the supported checks classNamePrefix = ""; + } else if (javaVersion < 23) { + classNamePrefix = "Java" + javaVersion; + } else { + // All Java version from 23 onwards will be able to use che checks in the Java23EntitlementChecker interface and implementation + classNamePrefix = "Java23"; } + final String className = packageName + "." + classNamePrefix + baseClassName; Class clazz; try { @@ -168,7 +532,25 @@ private static Class getVersionSpecificCheckerClass(int javaVersion, String p return clazz; } - private static final InstrumentationService INSTRUMENTER_FACTORY = new ProviderLocator<>( + private static ElasticsearchEntitlementChecker initChecker() { + final PolicyManager policyManager = createPolicyManager(); + + final Class clazz = getVersionSpecificCheckerClass(ElasticsearchEntitlementChecker.class, Runtime.version().feature()); + + Constructor constructor; + try { + constructor = clazz.getConstructor(PolicyManager.class); + } catch (NoSuchMethodException e) { + throw new AssertionError("entitlement impl is missing no arg constructor", e); + } + try { + return (ElasticsearchEntitlementChecker) constructor.newInstance(policyManager); + } catch (IllegalAccessException | InvocationTargetException | InstantiationException e) { + throw new AssertionError(e); + } + } + + private static final InstrumentationService INSTRUMENTATION_SERVICE = new ProviderLocator<>( "entitlement", InstrumentationService.class, "org.elasticsearch.entitlement.instrumentation", diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/EntitlementInstrumented.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/EntitlementInstrumented.java new file mode 100644 index 0000000000000..5621f8f6dcaf3 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/EntitlementInstrumented.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface EntitlementInstrumented { +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java index 66d8ad9488cfa..ece51a8414b70 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java @@ -9,14 +9,27 @@ package org.elasticsearch.entitlement.instrumentation; -import java.io.IOException; import java.util.Map; /** * The SPI service entry point for instrumentation. */ public interface InstrumentationService { + + String CHECK_METHOD_PREFIX = "check$"; + + record InstrumentationInfo(MethodKey targetMethod, CheckMethod checkMethod) {} + Instrumenter newInstrumenter(Class clazz, Map methods); - Map lookupMethods(Class clazz) throws IOException; + Map lookupMethods(Class clazz) throws ClassNotFoundException; + + InstrumentationInfo lookupImplementationMethod( + Class targetSuperclass, + String methodName, + Class implementationClass, + Class checkerClass, + String checkMethodName, + Class... parameterTypes + ) throws NoSuchMethodException, ClassNotFoundException; } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java index 9f39cbbbd0df0..c94dc70ae6262 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java @@ -10,5 +10,5 @@ package org.elasticsearch.entitlement.instrumentation; public interface Instrumenter { - byte[] instrumentClass(String className, byte[] classfileBuffer); + byte[] instrumentClass(String className, byte[] classfileBuffer, boolean verify); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java index c6512ee975dbf..6d4d4edaae162 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java @@ -20,12 +20,19 @@ public class Transformer implements ClassFileTransformer { private final Instrumenter instrumenter; private final Set classesToTransform; - public Transformer(Instrumenter instrumenter, Set classesToTransform) { + private boolean verifyClasses; + + public Transformer(Instrumenter instrumenter, Set classesToTransform, boolean verifyClasses) { this.instrumenter = instrumenter; this.classesToTransform = classesToTransform; + this.verifyClasses = verifyClasses; // TODO: Should warn if any MethodKey doesn't match any methods } + public void enableClassVerification() { + this.verifyClasses = true; + } + @Override public byte[] transform( ClassLoader loader, @@ -36,10 +43,10 @@ public byte[] transform( ) { if (classesToTransform.contains(className)) { // System.out.println("Transforming " + className); - return instrumenter.instrumentClass(className, classfileBuffer); + return instrumenter.instrumentClass(className, classfileBuffer, verifyClasses); } else { // System.out.println("Not transforming " + className); - return classfileBuffer; + return null; } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 3673bca0bed9c..a752f9c498e06 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -9,12 +9,18 @@ package org.elasticsearch.entitlement.runtime.api; +import jdk.nio.Channels; + import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.bridge.EntitlementChecker; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import java.io.File; +import java.io.FileDescriptor; +import java.io.FileFilter; +import java.io.FilenameFilter; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; @@ -22,8 +28,11 @@ import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.JarURLConnection; +import java.net.MalformedURLException; import java.net.MulticastSocket; import java.net.NetworkInterface; import java.net.Proxy; @@ -33,7 +42,10 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.SocketImplFactory; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; +import java.net.URLConnection; import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; import java.net.http.HttpClient; @@ -46,14 +58,43 @@ import java.nio.channels.DatagramChannel; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.nio.channels.spi.SelectorProvider; import java.nio.charset.Charset; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitor; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.WatchEvent; +import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.UserPrincipal; +import java.nio.file.spi.FileSystemProvider; +import java.security.KeyStore; +import java.security.Provider; import java.security.cert.CertStoreParameters; +import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ForkJoinPool; +import java.util.function.BiPredicate; +import java.util.logging.FileHandler; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -74,6 +115,11 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { this.policyManager = policyManager; } + /// ///////////////// + // + // Exit the JVM process + // + @Override public void check$java_lang_Runtime$exit(Class callerClass, Runtime runtime, int status) { policyManager.checkExitVM(callerClass); @@ -89,6 +135,11 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { policyManager.checkExitVM(callerClass); } + /// ///////////////// + // + // create class loaders + // + @Override public void check$java_lang_ClassLoader$(Class callerClass) { policyManager.checkCreateClassLoader(callerClass); @@ -105,51 +156,85 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { } @Override - public void check$java_security_SecureClassLoader$(Class callerClass) { + public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent) { + public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent) { + public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls) { + public void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent) { + public void check$java_net_URLClassLoader$( + Class callerClass, + String name, + URL[] urls, + ClassLoader parent, + URLStreamHandlerFactory factory + ) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory) { + public void check$java_security_SecureClassLoader$(Class callerClass) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent) { + public void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent) { policyManager.checkCreateClassLoader(callerClass); } @Override - public void check$java_net_URLClassLoader$( + public void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + /// ///////////////// + // + // "setFactory" methods + // + + @Override + public void check$javax_net_ssl_HttpsURLConnection$setSSLSocketFactory( Class callerClass, - String name, - URL[] urls, - ClassLoader parent, - URLStreamHandlerFactory factory + HttpsURLConnection connection, + SSLSocketFactory sf ) { - policyManager.checkCreateClassLoader(callerClass); + policyManager.checkSetHttpsConnectionProperties(callerClass); + } + + @Override + public void check$javax_net_ssl_HttpsURLConnection$$setDefaultSSLSocketFactory(Class callerClass, SSLSocketFactory sf) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void check$javax_net_ssl_HttpsURLConnection$$setDefaultHostnameVerifier(Class callerClass, HostnameVerifier hv) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void check$javax_net_ssl_SSLContext$$setDefault(Class callerClass, SSLContext context) { + policyManager.checkChangeJVMGlobalState(callerClass); } + /// ///////////////// + // + // Process creation + // + @Override public void check$java_lang_ProcessBuilder$start(Class callerClass, ProcessBuilder processBuilder) { policyManager.checkStartProcess(callerClass); @@ -160,6 +245,31 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { policyManager.checkStartProcess(callerClass); } + /// ///////////////// + // + // System Properties and similar + // + + @Override + public void check$java_lang_System$$clearProperty(Class callerClass, String key) { + policyManager.checkWriteProperty(callerClass, key); + } + + @Override + public void check$java_lang_System$$setProperties(Class callerClass, Properties props) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void check$java_lang_System$$setProperty(Class callerClass, String key, String value) { + policyManager.checkWriteProperty(callerClass, key); + } + + /// ///////////////// + // + // JVM-wide state changes + // + @Override public void check$java_lang_System$$setIn(Class callerClass, InputStream in) { policyManager.checkChangeJVMGlobalState(callerClass); @@ -211,27 +321,17 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { } @Override - public void check$com_sun_tools_jdi_VirtualMachineManagerImpl$$virtualMachineManager(Class callerClass) { + public void check$java_nio_charset_spi_CharsetProvider$(Class callerClass) { policyManager.checkChangeJVMGlobalState(callerClass); } @Override - public void check$java_lang_Thread$$setDefaultUncaughtExceptionHandler(Class callerClass, Thread.UncaughtExceptionHandler ueh) { + public void check$com_sun_tools_jdi_VirtualMachineManagerImpl$$virtualMachineManager(Class callerClass) { policyManager.checkChangeJVMGlobalState(callerClass); } @Override - public void check$java_lang_System$$clearProperty(Class callerClass, String key) { - policyManager.checkWriteProperty(callerClass, key); - } - - @Override - public void check$java_lang_System$$setProperty(Class callerClass, String key, String value) { - policyManager.checkWriteProperty(callerClass, key); - } - - @Override - public void check$java_lang_System$$setProperties(Class callerClass, Properties props) { + public void check$java_lang_Thread$$setDefaultUncaughtExceptionHandler(Class callerClass, Thread.UncaughtExceptionHandler ueh) { policyManager.checkChangeJVMGlobalState(callerClass); } @@ -350,29 +450,10 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { policyManager.checkChangeJVMGlobalState(callerClass); } - @Override - public void check$javax_net_ssl_HttpsURLConnection$setSSLSocketFactory( - Class callerClass, - HttpsURLConnection connection, - SSLSocketFactory sf - ) { - policyManager.checkSetHttpsConnectionProperties(callerClass); - } - - @Override - public void check$javax_net_ssl_HttpsURLConnection$$setDefaultSSLSocketFactory(Class callerClass, SSLSocketFactory sf) { - policyManager.checkChangeJVMGlobalState(callerClass); - } - - @Override - public void check$javax_net_ssl_HttpsURLConnection$$setDefaultHostnameVerifier(Class callerClass, HostnameVerifier hv) { - policyManager.checkChangeJVMGlobalState(callerClass); - } - - @Override - public void check$javax_net_ssl_SSLContext$$setDefault(Class callerClass, SSLContext context) { - policyManager.checkChangeJVMGlobalState(callerClass); - } + /// ///////////////// + // + // Network access + // @Override public void check$java_net_ProxySelector$$setDefault(Class callerClass, ProxySelector ps) { @@ -555,280 +636,2171 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { policyManager.checkOutboundNetworkAccess(callerClass); } + @SuppressWarnings("deprecation") + private URL extractJarFileUrl(URL jarUrl) { + String spec = jarUrl.getFile(); + int separator = spec.indexOf("!/"); + + // URL does not handle nested JAR URLs (it would be a MalformedURLException upon connection) + if (separator == -1) { + return null; + } + + try { + return new URL(spec.substring(0, separator)); + } catch (MalformedURLException e) { + return null; + } + } + + private boolean handleNetworkOrFileUrlCheck(Class callerClass, URL url) { + if (isNetworkUrl(url)) { + policyManager.checkOutboundNetworkAccess(callerClass); + return true; + } + if (isFileUrl(url)) { + checkURLFileRead(callerClass, url); + return true; + } + return false; + } + + private void checkJarURLAccess(Class callerClass, JarURLConnection that) { + var jarFileUrl = that.getJarFileURL(); + if (handleNetworkOrFileUrlCheck(callerClass, jarFileUrl)) { + return; + } + policyManager.checkUnsupportedURLProtocolConnection(callerClass, jarFileUrl.getProtocol()); + } + + private void checkEntitlementForUrl(Class callerClass, URL that) { + if (handleNetworkOrFileUrlCheck(callerClass, that)) { + return; + } + if (isJarUrl(that)) { + var jarFileUrl = extractJarFileUrl(that); + if (jarFileUrl == null || handleNetworkOrFileUrlCheck(callerClass, jarFileUrl) == false) { + policyManager.checkUnsupportedURLProtocolConnection(callerClass, "jar with unsupported inner protocol"); + } + } else { + policyManager.checkUnsupportedURLProtocolConnection(callerClass, that.getProtocol()); + } + } + + @Override + public void check$java_net_URL$openConnection(Class callerClass, java.net.URL that) { + checkEntitlementForUrl(callerClass, that); + } + @Override public void check$java_net_URL$openConnection(Class callerClass, URL that, Proxy proxy) { if (proxy.type() != Proxy.Type.DIRECT) { policyManager.checkOutboundNetworkAccess(callerClass); } + checkEntitlementForUrl(callerClass, that); } @Override - public void check$jdk_internal_net_http_HttpClientImpl$send( - Class callerClass, - HttpClient that, - HttpRequest request, - HttpResponse.BodyHandler responseBodyHandler - ) { - policyManager.checkOutboundNetworkAccess(callerClass); + public void check$java_net_URL$openStream(Class callerClass, java.net.URL that) { + checkEntitlementForUrl(callerClass, that); } @Override - public void check$jdk_internal_net_http_HttpClientImpl$sendAsync( - Class callerClass, - HttpClient that, - HttpRequest userRequest, - HttpResponse.BodyHandler responseHandler - ) { - policyManager.checkOutboundNetworkAccess(callerClass); + public void check$java_net_URL$getContent(Class callerClass, java.net.URL that) { + checkEntitlementForUrl(callerClass, that); } @Override - public void check$jdk_internal_net_http_HttpClientImpl$sendAsync( - Class callerClass, - HttpClient that, - HttpRequest userRequest, - HttpResponse.BodyHandler responseHandler, - HttpResponse.PushPromiseHandler pushPromiseHandler - ) { - policyManager.checkOutboundNetworkAccess(callerClass); + public void check$java_net_URL$getContent(Class callerClass, java.net.URL that, Class[] classes) { + checkEntitlementForUrl(callerClass, that); + } + + private static final Set NETWORK_PROTOCOLS = Set.of("http", "https", "ftp", "mailto"); + + private static boolean isNetworkUrl(java.net.URL url) { + return NETWORK_PROTOCOLS.contains(url.getProtocol()); + } + + private static boolean isFileUrl(java.net.URL url) { + return "file".equals(url.getProtocol()); + } + + private static boolean isJarUrl(java.net.URL url) { + return "jar".equals(url.getProtocol()); + } + + // We have to use class names for sun.net.www classes as java.base does not export them + private static final List ADDITIONAL_NETWORK_URL_CONNECT_CLASS_NAMES = List.of( + "sun.net.www.protocol.ftp.FtpURLConnection", + "sun.net.www.protocol.mailto.MailToURLConnection" + ); + + private static boolean isNetworkUrlConnection(java.net.URLConnection urlConnection) { + var connectionClass = urlConnection.getClass(); + return HttpURLConnection.class.isAssignableFrom(connectionClass) + || ADDITIONAL_NETWORK_URL_CONNECT_CLASS_NAMES.contains(connectionClass.getName()); + } + + // We have to use class names for sun.net.www classes as java.base does not export them + private static boolean isFileUrlConnection(java.net.URLConnection urlConnection) { + var connectionClass = urlConnection.getClass(); + return "sun.net.www.protocol.file.FileURLConnection".equals(connectionClass.getName()); + } + + private void checkEntitlementForURLConnection(Class callerClass, URLConnection that) { + if (isNetworkUrlConnection(that)) { + policyManager.checkOutboundNetworkAccess(callerClass); + } else if (isFileUrlConnection(that)) { + checkURLFileRead(callerClass, that.getURL()); + } else if (that instanceof JarURLConnection jarURLConnection) { + checkJarURLAccess(callerClass, jarURLConnection); + } else { + policyManager.checkUnsupportedURLProtocolConnection(callerClass, that.getURL().getProtocol()); + } } @Override - public void check$jdk_internal_net_http_HttpClientFacade$send( - Class callerClass, - HttpClient that, - HttpRequest request, - HttpResponse.BodyHandler responseBodyHandler - ) { - check$jdk_internal_net_http_HttpClientImpl$send(callerClass, that, request, responseBodyHandler); + public void check$java_net_URLConnection$getContentLength(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$jdk_internal_net_http_HttpClientFacade$sendAsync( - Class callerClass, - HttpClient that, - HttpRequest userRequest, - HttpResponse.BodyHandler responseHandler - ) { - check$jdk_internal_net_http_HttpClientImpl$sendAsync(callerClass, that, userRequest, responseHandler); + public void check$java_net_URLConnection$getContentLengthLong(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$jdk_internal_net_http_HttpClientFacade$sendAsync( - Class callerClass, - HttpClient that, - HttpRequest userRequest, - HttpResponse.BodyHandler responseHandler, - HttpResponse.PushPromiseHandler pushPromiseHandler - ) { - check$jdk_internal_net_http_HttpClientImpl$sendAsync(callerClass, that, userRequest, responseHandler, pushPromiseHandler); + public void check$java_net_URLConnection$getContentType(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$java_security_cert_CertStore$$getInstance(Class callerClass, String type, CertStoreParameters params) { - // We need to check "just" the LDAPCertStore instantiation: this is the CertStore that will try to perform a network operation - // (connect to an LDAP server). But LDAPCertStore is internal (created via SPI), so we instrument the general factory instead and - // then do the check only for the path that leads to sensitive code (by looking at the `type` parameter). - if ("LDAP".equals(type)) { - policyManager.checkOutboundNetworkAccess(callerClass); - } + public void check$java_net_URLConnection$getContentEncoding(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$java_nio_channels_AsynchronousServerSocketChannel$bind( - Class callerClass, - AsynchronousServerSocketChannel that, - SocketAddress local - ) { - policyManager.checkInboundNetworkAccess(callerClass); + public void check$java_net_URLConnection$getExpiration(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_AsynchronousServerSocketChannelImpl$bind( + public void check$java_net_URLConnection$getDate(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); + } + + @Override + public void check$java_net_URLConnection$getLastModified(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); + } + + @Override + public void check$java_net_URLConnection$getHeaderFieldInt( Class callerClass, - AsynchronousServerSocketChannel that, - SocketAddress local, - int backlog + java.net.URLConnection that, + String name, + int defaultValue ) { - policyManager.checkInboundNetworkAccess(callerClass); + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_AsynchronousSocketChannelImpl$bind( + public void check$java_net_URLConnection$getHeaderFieldLong( Class callerClass, - AsynchronousSocketChannel that, - SocketAddress local + java.net.URLConnection that, + String name, + long defaultValue ) { - policyManager.checkInboundNetworkAccess(callerClass); + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_DatagramChannelImpl$bind(Class callerClass, DatagramChannel that, SocketAddress local) { - policyManager.checkInboundNetworkAccess(callerClass); + public void check$java_net_URLConnection$getHeaderFieldDate( + Class callerClass, + java.net.URLConnection that, + String name, + long defaultValue + ) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$java_nio_channels_ServerSocketChannel$bind(Class callerClass, ServerSocketChannel that, SocketAddress local) { - policyManager.checkInboundNetworkAccess(callerClass); + public void check$java_net_URLConnection$getContent(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_ServerSocketChannelImpl$bind( - Class callerClass, - ServerSocketChannel that, - SocketAddress local, - int backlog - ) { - policyManager.checkInboundNetworkAccess(callerClass); + public void check$java_net_URLConnection$getContent(Class callerClass, java.net.URLConnection that, Class[] classes) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_SocketChannelImpl$bind(Class callerClass, SocketChannel that, SocketAddress local) { + public void check$java_net_HttpURLConnection$getResponseCode(Class callerClass, java.net.HttpURLConnection that) { policyManager.checkOutboundNetworkAccess(callerClass); } @Override - public void check$sun_nio_ch_SocketChannelImpl$connect(Class callerClass, SocketChannel that, SocketAddress remote) { + public void check$java_net_HttpURLConnection$getResponseMessage(Class callerClass, java.net.HttpURLConnection that) { policyManager.checkOutboundNetworkAccess(callerClass); } @Override - public void check$sun_nio_ch_AsynchronousSocketChannelImpl$connect( + public void check$java_net_HttpURLConnection$getHeaderFieldDate( Class callerClass, - AsynchronousSocketChannel that, - SocketAddress remote + java.net.HttpURLConnection that, + String name, + long defaultValue ) { policyManager.checkOutboundNetworkAccess(callerClass); } + // Using java.net.URLConnection for "that" as sun.net.www.URLConnection is not exported @Override - public void check$sun_nio_ch_AsynchronousSocketChannelImpl$connect( - Class callerClass, - AsynchronousSocketChannel that, - SocketAddress remote, - Object attachment, - CompletionHandler handler - ) { - policyManager.checkOutboundNetworkAccess(callerClass); + public void check$sun_net_www_URLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, String name) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_DatagramChannelImpl$connect(Class callerClass, DatagramChannel that, SocketAddress remote) { - policyManager.checkOutboundNetworkAccess(callerClass); + public void check$sun_net_www_URLConnection$getHeaderFields(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_ServerSocketChannelImpl$accept(Class callerClass, ServerSocketChannel that) { - policyManager.checkInboundNetworkAccess(callerClass); + public void check$sun_net_www_URLConnection$getHeaderFieldKey(Class callerClass, java.net.URLConnection that, int n) { + checkEntitlementForURLConnection(callerClass, that); } @Override - public void check$sun_nio_ch_AsynchronousServerSocketChannelImpl$accept(Class callerClass, AsynchronousServerSocketChannel that) { + public void check$sun_net_www_URLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, int n) { + checkEntitlementForURLConnection(callerClass, that); + } + + @Override + public void check$sun_net_www_URLConnection$getContentType(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); + } + + @Override + public void check$sun_net_www_URLConnection$getContentLength(Class callerClass, java.net.URLConnection that) { + checkEntitlementForURLConnection(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_ftp_FtpURLConnection$connect(Class callerClass, java.net.URLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_ftp_FtpURLConnection$getInputStream(Class callerClass, java.net.URLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_ftp_FtpURLConnection$getOutputStream(Class callerClass, java.net.URLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$$openConnectionCheckRedirects( + Class callerClass, + java.net.URLConnection c + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$connect(Class callerClass, java.net.HttpURLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getOutputStream(Class callerClass, java.net.HttpURLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getInputStream(Class callerClass, java.net.HttpURLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getErrorStream(Class callerClass, java.net.HttpURLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderField( + Class callerClass, + java.net.HttpURLConnection that, + String name + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderFields(Class callerClass, java.net.HttpURLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderField( + Class callerClass, + java.net.HttpURLConnection that, + int n + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_http_HttpURLConnection$getHeaderFieldKey( + Class callerClass, + java.net.HttpURLConnection that, + int n + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$connect( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getOutputStream( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getInputStream( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getErrorStream( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderField( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFields( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderField( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + int n + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldKey( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + int n + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getResponseCode( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getResponseMessage( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentLength( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentLengthLong( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentType( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContentEncoding( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getExpiration( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getDate( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getLastModified( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldInt( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name, + int defaultValue + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldLong( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name, + long defaultValue + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getHeaderFieldDate( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + String name, + long defaultValue + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContent( + Class callerClass, + javax.net.ssl.HttpsURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_HttpsURLConnectionImpl$getContent( + Class callerClass, + javax.net.ssl.HttpsURLConnection that, + Class[] classes + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_https_AbstractDelegateHttpsURLConnection$connect( + Class callerClass, + java.net.HttpURLConnection that + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_mailto_MailToURLConnection$connect(Class callerClass, java.net.URLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_net_www_protocol_mailto_MailToURLConnection$getOutputStream(Class callerClass, java.net.URLConnection that) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$jdk_internal_net_http_HttpClientImpl$send( + Class callerClass, + HttpClient that, + HttpRequest request, + HttpResponse.BodyHandler responseBodyHandler + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$jdk_internal_net_http_HttpClientImpl$sendAsync( + Class callerClass, + HttpClient that, + HttpRequest userRequest, + HttpResponse.BodyHandler responseHandler + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$jdk_internal_net_http_HttpClientImpl$sendAsync( + Class callerClass, + HttpClient that, + HttpRequest userRequest, + HttpResponse.BodyHandler responseHandler, + HttpResponse.PushPromiseHandler pushPromiseHandler + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$jdk_internal_net_http_HttpClientFacade$send( + Class callerClass, + HttpClient that, + HttpRequest request, + HttpResponse.BodyHandler responseBodyHandler + ) { + check$jdk_internal_net_http_HttpClientImpl$send(callerClass, that, request, responseBodyHandler); + } + + @Override + public void check$jdk_internal_net_http_HttpClientFacade$sendAsync( + Class callerClass, + HttpClient that, + HttpRequest userRequest, + HttpResponse.BodyHandler responseHandler + ) { + check$jdk_internal_net_http_HttpClientImpl$sendAsync(callerClass, that, userRequest, responseHandler); + } + + @Override + public void check$jdk_internal_net_http_HttpClientFacade$sendAsync( + Class callerClass, + HttpClient that, + HttpRequest userRequest, + HttpResponse.BodyHandler responseHandler, + HttpResponse.PushPromiseHandler pushPromiseHandler + ) { + check$jdk_internal_net_http_HttpClientImpl$sendAsync(callerClass, that, userRequest, responseHandler, pushPromiseHandler); + } + + @Override + public void check$java_security_cert_CertStore$$getInstance(Class callerClass, String type, CertStoreParameters params) { + // We need to check "just" the LDAPCertStore instantiation: this is the CertStore that will try to perform a network operation + // (connect to an LDAP server). But LDAPCertStore is internal (created via SPI), so we instrument the general factory instead and + // then do the check only for the path that leads to sensitive code (by looking at the `type` parameter). + if ("LDAP".equals(type)) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + } + + @Override + public void check$java_nio_channels_AsynchronousServerSocketChannel$bind( + Class callerClass, + AsynchronousServerSocketChannel that, + SocketAddress local + ) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_AsynchronousServerSocketChannelImpl$bind( + Class callerClass, + AsynchronousServerSocketChannel that, + SocketAddress local, + int backlog + ) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_AsynchronousSocketChannelImpl$bind( + Class callerClass, + AsynchronousSocketChannel that, + SocketAddress local + ) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_DatagramChannelImpl$bind(Class callerClass, DatagramChannel that, SocketAddress local) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$java_nio_channels_ServerSocketChannel$bind(Class callerClass, ServerSocketChannel that, SocketAddress local) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_ServerSocketChannelImpl$bind( + Class callerClass, + ServerSocketChannel that, + SocketAddress local, + int backlog + ) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_SocketChannelImpl$bind(Class callerClass, SocketChannel that, SocketAddress local) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_SocketChannelImpl$connect(Class callerClass, SocketChannel that, SocketAddress remote) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_AsynchronousSocketChannelImpl$connect( + Class callerClass, + AsynchronousSocketChannel that, + SocketAddress remote + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_AsynchronousSocketChannelImpl$connect( + Class callerClass, + AsynchronousSocketChannel that, + SocketAddress remote, + Object attachment, + CompletionHandler handler + ) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_DatagramChannelImpl$connect(Class callerClass, DatagramChannel that, SocketAddress remote) { + policyManager.checkOutboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_ServerSocketChannelImpl$accept(Class callerClass, ServerSocketChannel that) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_AsynchronousServerSocketChannelImpl$accept(Class callerClass, AsynchronousServerSocketChannel that) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_AsynchronousServerSocketChannelImpl$accept( + Class callerClass, + AsynchronousServerSocketChannel that, + Object attachment, + CompletionHandler handler + ) { + policyManager.checkInboundNetworkAccess(callerClass); + } + + @Override + public void check$sun_nio_ch_DatagramChannelImpl$send( + Class callerClass, + DatagramChannel that, + ByteBuffer src, + SocketAddress target + ) { + if (target instanceof InetSocketAddress isa && isa.getAddress().isMulticastAddress()) { + policyManager.checkAllNetworkAccess(callerClass); + } else { + policyManager.checkOutboundNetworkAccess(callerClass); + } + } + + @Override + public void check$sun_nio_ch_DatagramChannelImpl$receive(Class callerClass, DatagramChannel that, ByteBuffer dst) { policyManager.checkInboundNetworkAccess(callerClass); } @Override - public void check$sun_nio_ch_AsynchronousServerSocketChannelImpl$accept( + public void check$java_nio_channels_spi_SelectorProvider$(Class callerClass) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_nio_channels_spi_AsynchronousChannelProvider$(Class callerClass) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void checkSelectorProviderInheritedChannel(Class callerClass, SelectorProvider that) { + policyManager.checkChangeNetworkHandling(callerClass); + } + + @Override + public void check$java_lang_Runtime$load(Class callerClass, Runtime that, String filename) { + policyManager.checkFileRead(callerClass, Path.of(filename)); + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_Runtime$loadLibrary(Class callerClass, Runtime that, String libname) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_System$$load(Class callerClass, String filename) { + policyManager.checkFileRead(callerClass, Path.of(filename)); + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_System$$loadLibrary(Class callerClass, String libname) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_ModuleLayer$Controller$enableNativeAccess( + Class callerClass, + ModuleLayer.Controller that, + Module target + ) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + /// ///////////////// + // + // File access + // + + // old io (ie File) + + @Override + public void check$java_io_File$canExecute(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$canRead(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$canWrite(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$createNewFile(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$$createTempFile(Class callerClass, String prefix, String suffix, File directory) { + policyManager.checkFileWrite(callerClass, directory); + } + + @Override + public void check$java_io_File$delete(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$deleteOnExit(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$exists(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$isDirectory(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$isFile(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$isHidden(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$lastModified(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$length(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$list(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$list(Class callerClass, File file, FilenameFilter filter) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$listFiles(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$listFiles(Class callerClass, File file, FileFilter filter) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$listFiles(Class callerClass, File file, FilenameFilter filter) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_File$mkdir(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$mkdirs(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$renameTo(Class callerClass, File file, File dest) { + policyManager.checkFileRead(callerClass, file); + policyManager.checkFileWrite(callerClass, dest); + } + + @Override + public void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setExecutable(Class callerClass, File file, boolean executable, boolean ownerOnly) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setLastModified(Class callerClass, File file, long time) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setReadable(Class callerClass, File file, boolean readable) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setReadable(Class callerClass, File file, boolean readable, boolean ownerOnly) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setReadOnly(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setWritable(Class callerClass, File file, boolean writable) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_File$setWritable(Class callerClass, File file, boolean writable, boolean ownerOnly) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileInputStream$(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_FileInputStream$(Class callerClass, FileDescriptor fd) { + policyManager.checkFileDescriptorRead(callerClass); + } + + @Override + public void check$java_io_FileInputStream$(Class callerClass, String name) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileOutputStream$(Class callerClass, String name) { + policyManager.checkFileWrite(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileOutputStream$(Class callerClass, String name, boolean append) { + policyManager.checkFileWrite(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileOutputStream$(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileOutputStream$(Class callerClass, File file, boolean append) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileOutputStream$(Class callerClass, FileDescriptor fd) { + policyManager.checkFileDescriptorWrite(callerClass); + } + + @Override + public void check$java_io_FileReader$(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_FileReader$(Class callerClass, File file, Charset charset) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_io_FileReader$(Class callerClass, FileDescriptor fd) { + policyManager.checkFileDescriptorRead(callerClass); + } + + @Override + public void check$java_io_FileReader$(Class callerClass, String name) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileReader$(Class callerClass, String name, Charset charset) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, File file) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, File file, boolean append) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, File file, Charset charset) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, File file, Charset charset, boolean append) { + policyManager.checkFileWrite(callerClass, file); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, FileDescriptor fd) { + policyManager.checkFileDescriptorWrite(callerClass); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, String name) { + policyManager.checkFileWrite(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, String name, boolean append) { + policyManager.checkFileWrite(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, String name, Charset charset) { + policyManager.checkFileWrite(callerClass, new File(name)); + } + + @Override + public void check$java_io_FileWriter$(Class callerClass, String name, Charset charset, boolean append) { + policyManager.checkFileWrite(callerClass, new File(name)); + } + + @Override + public void check$java_io_RandomAccessFile$(Class callerClass, String name, String mode) { + if (mode.equals("r")) { + policyManager.checkFileRead(callerClass, new File(name)); + } else { + policyManager.checkFileWrite(callerClass, new File(name)); + } + } + + @Override + public void check$java_io_RandomAccessFile$(Class callerClass, File file, String mode) { + if (mode.equals("r")) { + policyManager.checkFileRead(callerClass, file); + } else { + policyManager.checkFileWrite(callerClass, file); + } + } + + @Override + public void check$java_security_KeyStore$$getInstance(Class callerClass, File file, char[] password) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_security_KeyStore$$getInstance(Class callerClass, File file, KeyStore.LoadStoreParameter param) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_security_KeyStore$Builder$$newInstance( + Class callerClass, + File file, + KeyStore.ProtectionParameter protection + ) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_security_KeyStore$Builder$$newInstance( + Class callerClass, + String type, + Provider provider, + File file, + KeyStore.ProtectionParameter protection + ) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_util_Scanner$(Class callerClass, File source) { + policyManager.checkFileRead(callerClass, source); + } + + @Override + public void check$java_util_Scanner$(Class callerClass, File source, String charsetName) { + policyManager.checkFileRead(callerClass, source); + } + + @Override + public void check$java_util_Scanner$(Class callerClass, File source, Charset charset) { + policyManager.checkFileRead(callerClass, source); + } + + @Override + public void check$java_util_jar_JarFile$(Class callerClass, String name) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_util_jar_JarFile$(Class callerClass, String name, boolean verify) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_util_jar_JarFile$(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_util_jar_JarFile$(Class callerClass, File file, boolean verify) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_util_jar_JarFile$(Class callerClass, File file, boolean verify, int mode) { + policyManager.checkFileWithZipMode(callerClass, file, mode); + } + + @Override + public void check$java_util_jar_JarFile$(Class callerClass, File file, boolean verify, int mode, Runtime.Version version) { + policyManager.checkFileWithZipMode(callerClass, file, mode); + } + + @Override + public void check$java_util_zip_ZipFile$(Class callerClass, String name) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_util_zip_ZipFile$(Class callerClass, String name, Charset charset) { + policyManager.checkFileRead(callerClass, new File(name)); + } + + @Override + public void check$java_util_zip_ZipFile$(Class callerClass, File file) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_util_zip_ZipFile$(Class callerClass, File file, int mode) { + policyManager.checkFileWithZipMode(callerClass, file, mode); + } + + @Override + public void check$java_util_zip_ZipFile$(Class callerClass, File file, Charset charset) { + policyManager.checkFileRead(callerClass, file); + } + + @Override + public void check$java_util_zip_ZipFile$(Class callerClass, File file, int mode, Charset charset) { + policyManager.checkFileWithZipMode(callerClass, file, mode); + } + + // nio + + @Override + public void check$java_nio_channels_FileChannel$(Class callerClass) { + policyManager.checkChangeFilesHandling(callerClass); + } + + @Override + public void check$java_nio_channels_FileChannel$$open( + Class callerClass, + Path path, + Set options, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void check$java_nio_channels_FileChannel$$open(Class callerClass, Path path, OpenOption... options) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void check$java_nio_channels_AsynchronousFileChannel$(Class callerClass) { + policyManager.checkChangeFilesHandling(callerClass); + } + + @Override + public void check$java_nio_channels_AsynchronousFileChannel$$open( + Class callerClass, + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void check$java_nio_channels_AsynchronousFileChannel$$open(Class callerClass, Path path, OpenOption... options) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void check$jdk_nio_Channels$$readWriteSelectableChannel( + Class callerClass, + FileDescriptor fd, + Channels.SelectableChannelCloser closer + ) { + policyManager.checkFileDescriptorWrite(callerClass); + } + + @Override + public void check$java_nio_file_Files$$getOwner(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$probeContentType(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$setOwner(Class callerClass, Path path, UserPrincipal principal) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$newInputStream(Class callerClass, Path path, OpenOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$newOutputStream(Class callerClass, Path path, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$newByteChannel( + Class callerClass, + Path path, + Set options, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void check$java_nio_file_Files$$newByteChannel(Class callerClass, Path path, OpenOption... options) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } + } + + @Override + public void check$java_nio_file_Files$$newDirectoryStream(Class callerClass, Path dir) { + policyManager.checkFileRead(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$newDirectoryStream(Class callerClass, Path dir, String glob) { + policyManager.checkFileRead(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$newDirectoryStream(Class callerClass, Path dir, DirectoryStream.Filter filter) { + policyManager.checkFileRead(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$createFile(Class callerClass, Path path, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$createDirectory(Class callerClass, Path dir, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$createDirectories(Class callerClass, Path dir, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$createTempFile( + Class callerClass, + Path dir, + String prefix, + String suffix, + FileAttribute... attrs + ) { + policyManager.checkFileWrite(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$createTempFile(Class callerClass, String prefix, String suffix, FileAttribute... attrs) { + policyManager.checkCreateTempFile(callerClass); + } + + @Override + public void check$java_nio_file_Files$$createTempDirectory(Class callerClass, Path dir, String prefix, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$createTempDirectory(Class callerClass, String prefix, FileAttribute... attrs) { + policyManager.checkCreateTempFile(callerClass); + } + + private static Path resolveLinkTarget(Path path, Path target) { + var parent = path.getParent(); + return parent == null ? target : parent.resolve(target); + } + + @Override + public void check$java_nio_file_Files$$createSymbolicLink(Class callerClass, Path link, Path target, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, link); + policyManager.checkFileRead(callerClass, resolveLinkTarget(link, target)); + } + + @Override + public void check$java_nio_file_Files$$createLink(Class callerClass, Path link, Path existing) { + policyManager.checkFileWrite(callerClass, link); + policyManager.checkFileRead(callerClass, resolveLinkTarget(link, existing)); + } + + @Override + public void check$java_nio_file_Files$$delete(Class callerClass, Path path) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$deleteIfExists(Class callerClass, Path path) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$copy(Class callerClass, Path source, Path target, CopyOption... options) { + policyManager.checkFileRead(callerClass, source); + policyManager.checkFileWrite(callerClass, target); + } + + @Override + public void check$java_nio_file_Files$$move(Class callerClass, Path source, Path target, CopyOption... options) { + policyManager.checkFileWrite(callerClass, source); + policyManager.checkFileWrite(callerClass, target); + } + + @Override + public void check$java_nio_file_Files$$readSymbolicLink(Class callerClass, Path link) { + policyManager.checkFileRead(callerClass, link); + } + + @Override + public void check$java_nio_file_Files$$getFileStore(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isSameFile(Class callerClass, Path path, Path path2) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkFileRead(callerClass, path2); + } + + @Override + public void check$java_nio_file_Files$$mismatch(Class callerClass, Path path, Path path2) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkFileRead(callerClass, path2); + } + + @Override + public void check$java_nio_file_Files$$isHidden(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$getFileAttributeView( + Class callerClass, + Path path, + Class type, + LinkOption... options + ) { + policyManager.checkGetFileAttributeView(callerClass); + } + + @Override + public void check$java_nio_file_Files$$readAttributes( + Class callerClass, + Path path, + Class type, + LinkOption... options + ) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$setAttribute( + Class callerClass, + Path path, + String attribute, + Object value, + LinkOption... options + ) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$getAttribute(Class callerClass, Path path, String attribute, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$readAttributes(Class callerClass, Path path, String attributes, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$getPosixFilePermissions(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$setPosixFilePermissions(Class callerClass, Path path, Set perms) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isSymbolicLink(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isDirectory(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isRegularFile(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$getLastModifiedTime(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$setLastModifiedTime(Class callerClass, Path path, FileTime time) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$size(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$exists(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$notExists(Class callerClass, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isReadable(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isWritable(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$isExecutable(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$walkFileTree( + Class callerClass, + Path start, + Set options, + int maxDepth, + FileVisitor visitor + ) { + policyManager.checkFileRead(callerClass, start); + } + + @Override + public void check$java_nio_file_Files$$walkFileTree(Class callerClass, Path start, FileVisitor visitor) { + policyManager.checkFileRead(callerClass, start); + } + + @Override + public void check$java_nio_file_Files$$newBufferedReader(Class callerClass, Path path, Charset cs) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$newBufferedReader(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$newBufferedWriter(Class callerClass, Path path, Charset cs, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$newBufferedWriter(Class callerClass, Path path, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$copy(Class callerClass, InputStream in, Path target, CopyOption... options) { + policyManager.checkFileWrite(callerClass, target); + } + + @Override + public void check$java_nio_file_Files$$copy(Class callerClass, Path source, OutputStream out) { + policyManager.checkFileRead(callerClass, source); + } + + @Override + public void check$java_nio_file_Files$$readAllBytes(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$readString(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$readString(Class callerClass, Path path, Charset cs) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$readAllLines(Class callerClass, Path path, Charset cs) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$readAllLines(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$write(Class callerClass, Path path, byte[] bytes, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$write( + Class callerClass, + Path path, + Iterable lines, + Charset cs, + OpenOption... options + ) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$write( + Class callerClass, + Path path, + Iterable lines, + OpenOption... options + ) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$writeString(Class callerClass, Path path, CharSequence csq, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$writeString( + Class callerClass, + Path path, + CharSequence csq, + Charset cs, + OpenOption... options + ) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$list(Class callerClass, Path dir) { + policyManager.checkFileRead(callerClass, dir); + } + + @Override + public void check$java_nio_file_Files$$walk(Class callerClass, Path start, int maxDepth, FileVisitOption... options) { + policyManager.checkFileRead(callerClass, start); + } + + @Override + public void check$java_nio_file_Files$$walk(Class callerClass, Path start, FileVisitOption... options) { + policyManager.checkFileRead(callerClass, start); + } + + @Override + public void check$java_nio_file_Files$$find( + Class callerClass, + Path start, + int maxDepth, + BiPredicate matcher, + FileVisitOption... options + ) { + policyManager.checkFileRead(callerClass, start); + } + + @Override + public void check$java_nio_file_Files$$lines(Class callerClass, Path path, Charset cs) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_nio_file_Files$$lines(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + // file system providers + + @Override + public void check$java_nio_file_spi_FileSystemProvider$(Class callerClass) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, boolean append) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count, boolean append) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, long limit, int count, boolean append) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$close(Class callerClass, FileHandler that) { + // Note that there's no IT test for this one, because there's no way to create + // a FileHandler. However, we have this check just in case someone does manage + // to get their hands on a FileHandler and uses close() to cause its lock file to be deleted. + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload( Class callerClass, - AsynchronousServerSocketChannel that, - Object attachment, - CompletionHandler handler + Path directory, + OpenOption... openOptions ) { - policyManager.checkInboundNetworkAccess(callerClass); + policyManager.checkFileWrite(callerClass, directory); } @Override - public void check$sun_nio_ch_DatagramChannelImpl$send( + public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory) { + policyManager.checkFileWrite(callerClass, directory); + } + + @Override + public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory, OpenOption... openOptions) { + policyManager.checkFileWrite(callerClass, directory); + } + + @Override + public void checkNewFileSystem(Class callerClass, FileSystemProvider that, URI uri, Map env) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void checkNewFileSystem(Class callerClass, FileSystemProvider that, Path path, Map env) { + policyManager.checkChangeJVMGlobalState(callerClass); + } + + @Override + public void checkNewInputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkNewOutputStream(Class callerClass, FileSystemProvider that, Path path, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + private static boolean isOpenForWrite(Set options) { + return options.contains(StandardOpenOption.WRITE) + || options.contains(StandardOpenOption.APPEND) + || options.contains(StandardOpenOption.CREATE) + || options.contains(StandardOpenOption.CREATE_NEW) + || options.contains(StandardOpenOption.DELETE_ON_CLOSE); + } + + private static boolean isOpenForWrite(OpenOption... options) { + return Arrays.stream(options) + .anyMatch( + o -> o.equals(StandardOpenOption.WRITE) + || o.equals(StandardOpenOption.APPEND) + || o.equals(StandardOpenOption.CREATE) + || o.equals(StandardOpenOption.CREATE_NEW) + || o.equals(StandardOpenOption.DELETE_ON_CLOSE) + ); + } + + @Override + public void checkNewFileChannel( Class callerClass, - DatagramChannel that, - ByteBuffer src, - SocketAddress target + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs ) { - if (target instanceof InetSocketAddress isa && isa.getAddress().isMulticastAddress()) { - policyManager.checkAllNetworkAccess(callerClass); + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); } else { - policyManager.checkOutboundNetworkAccess(callerClass); + policyManager.checkFileRead(callerClass, path); } } @Override - public void check$sun_nio_ch_DatagramChannelImpl$receive(Class callerClass, DatagramChannel that, ByteBuffer dst) { - policyManager.checkInboundNetworkAccess(callerClass); + public void checkNewAsynchronousFileChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } } @Override - public void check$java_lang_Runtime$load(Class callerClass, Runtime that, String filename) { - // TODO: check filesystem entitlement READ - policyManager.checkLoadingNativeLibraries(callerClass); + public void checkNewByteChannel( + Class callerClass, + FileSystemProvider that, + Path path, + Set options, + FileAttribute... attrs + ) { + if (isOpenForWrite(options)) { + policyManager.checkFileWrite(callerClass, path); + } else { + policyManager.checkFileRead(callerClass, path); + } } @Override - public void check$java_lang_Runtime$loadLibrary(Class callerClass, Runtime that, String libname) { - policyManager.checkLoadingNativeLibraries(callerClass); + public void checkNewDirectoryStream( + Class callerClass, + FileSystemProvider that, + Path dir, + DirectoryStream.Filter filter + ) { + policyManager.checkFileRead(callerClass, dir); } @Override - public void check$java_lang_System$$load(Class callerClass, String filename) { - // TODO: check filesystem entitlement READ - policyManager.checkLoadingNativeLibraries(callerClass); + public void checkCreateDirectory(Class callerClass, FileSystemProvider that, Path dir, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, dir); } @Override - public void check$java_lang_System$$loadLibrary(Class callerClass, String libname) { - policyManager.checkLoadingNativeLibraries(callerClass); + public void checkCreateSymbolicLink(Class callerClass, FileSystemProvider that, Path link, Path target, FileAttribute... attrs) { + policyManager.checkFileWrite(callerClass, link); + policyManager.checkFileRead(callerClass, resolveLinkTarget(link, target)); } @Override - public void check$java_lang_ModuleLayer$Controller$enableNativeAccess( - Class callerClass, - ModuleLayer.Controller that, - Module target - ) { - policyManager.checkLoadingNativeLibraries(callerClass); + public void checkCreateLink(Class callerClass, FileSystemProvider that, Path link, Path existing) { + policyManager.checkFileWrite(callerClass, link); + policyManager.checkFileRead(callerClass, resolveLinkTarget(link, existing)); } @Override - public void check$java_util_Scanner$(Class callerClass, File source) { - policyManager.checkFileRead(callerClass, source); + public void checkDelete(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileWrite(callerClass, path); } @Override - public void check$java_util_Scanner$(Class callerClass, File source, String charsetName) { - policyManager.checkFileRead(callerClass, source); + public void checkDeleteIfExists(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileWrite(callerClass, path); } @Override - public void check$java_util_Scanner$(Class callerClass, File source, Charset charset) { + public void checkReadSymbolicLink(Class callerClass, FileSystemProvider that, Path link) { + policyManager.checkFileRead(callerClass, link); + } + + @Override + public void checkCopy(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options) { + policyManager.checkFileWrite(callerClass, target); policyManager.checkFileRead(callerClass, source); } @Override - public void check$java_io_FileOutputStream$(Class callerClass, String name) { - policyManager.checkFileWrite(callerClass, new File(name)); + public void checkMove(Class callerClass, FileSystemProvider that, Path source, Path target, CopyOption... options) { + policyManager.checkFileWrite(callerClass, target); + policyManager.checkFileWrite(callerClass, source); } @Override - public void check$java_io_FileOutputStream$(Class callerClass, String name, boolean append) { - policyManager.checkFileWrite(callerClass, new File(name)); + public void checkIsSameFile(Class callerClass, FileSystemProvider that, Path path, Path path2) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkFileRead(callerClass, path2); } @Override - public void check$java_io_FileOutputStream$(Class callerClass, File file) { - policyManager.checkFileWrite(callerClass, file); + public void checkIsHidden(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileRead(callerClass, path); } @Override - public void check$java_io_FileOutputStream$(Class callerClass, File file, boolean append) { - policyManager.checkFileWrite(callerClass, file); + public void checkGetFileStore(Class callerClass, FileSystemProvider that, Path path) { + policyManager.checkFileRead(callerClass, path); } @Override - public void check$java_nio_file_Files$$probeContentType(Class callerClass, Path path) { + public void checkCheckAccess(Class callerClass, FileSystemProvider that, Path path, AccessMode... modes) { policyManager.checkFileRead(callerClass, path); } @Override - public void check$java_nio_file_Files$$setOwner(Class callerClass, Path path, UserPrincipal principal) { + public void checkGetFileAttributeView(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options) { + policyManager.checkGetFileAttributeView(callerClass); + } + + @Override + public void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, Class type, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkReadAttributes(Class callerClass, FileSystemProvider that, Path path, String attributes, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkSetAttribute( + Class callerClass, + FileSystemProvider that, + Path path, + String attribute, + Object value, + LinkOption... options + ) { policyManager.checkFileWrite(callerClass, path); + + } + + // Thread management + + @Override + public void check$java_lang_Thread$start(Class callerClass, Thread thread) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setDaemon(Class callerClass, Thread thread, boolean on) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_ThreadGroup$setDaemon(Class callerClass, ThreadGroup threadGroup, boolean daemon) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_util_concurrent_ForkJoinPool$setParallelism(Class callerClass, ForkJoinPool forkJoinPool, int size) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setName(Class callerClass, Thread thread, String name) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setPriority(Class callerClass, Thread thread, int newPriority) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_Thread$setUncaughtExceptionHandler( + Class callerClass, + Thread thread, + Thread.UncaughtExceptionHandler ueh + ) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void check$java_lang_ThreadGroup$setMaxPriority(Class callerClass, ThreadGroup threadGroup, int pri) { + policyManager.checkManageThreadsEntitlement(callerClass); + } + + @Override + public void checkGetFileStoreAttributeView(Class callerClass, FileStore that, Class type) { + policyManager.checkWriteStoreAttributes(callerClass); + } + + @Override + public void checkGetAttribute(Class callerClass, FileStore that, String attribute) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetBlockSize(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetTotalSpace(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetUnallocatedSpace(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkGetUsableSpace(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkIsReadOnly(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkName(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkType(Class callerClass, FileStore that) { + policyManager.checkReadStoreAttributes(callerClass); + } + + @Override + public void checkPathToRealPath(Class callerClass, Path that, LinkOption... options) throws NoSuchFileException { + boolean followLinks = true; + for (LinkOption option : options) { + if (option == LinkOption.NOFOLLOW_LINKS) { + followLinks = false; + } + } + policyManager.checkFileRead(callerClass, that, followLinks); + } + + @Override + public void checkPathRegister(Class callerClass, Path that, WatchService watcher, WatchEvent.Kind... events) { + policyManager.checkFileRead(callerClass, that); + } + + @Override + public void checkPathRegister( + Class callerClass, + Path that, + WatchService watcher, + WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers + ) { + policyManager.checkFileRead(callerClass, that); + } + + private void checkURLFileRead(Class callerClass, URL url) { + try { + policyManager.checkFileRead(callerClass, Paths.get(url.toURI())); + } catch (URISyntaxException e) { + // We expect this method to be called only on File URLs; otherwise the underlying method would fail anyway + throw new RuntimeException(e); + } + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$connect(Class callerClass, java.net.URLConnection that) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getHeaderFields(Class callerClass, java.net.URLConnection that) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getHeaderField( + Class callerClass, + java.net.URLConnection that, + String name + ) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getHeaderField(Class callerClass, java.net.URLConnection that, int n) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getContentLength(Class callerClass, java.net.URLConnection that) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getContentLengthLong(Class callerClass, java.net.URLConnection that) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getHeaderFieldKey( + Class callerClass, + java.net.URLConnection that, + int n + ) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getLastModified(Class callerClass, java.net.URLConnection that) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$sun_net_www_protocol_file_FileURLConnection$getInputStream(Class callerClass, java.net.URLConnection that) { + checkURLFileRead(callerClass, that.getURL()); + } + + @Override + public void check$java_net_JarURLConnection$getManifest(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$java_net_JarURLConnection$getJarEntry(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$java_net_JarURLConnection$getAttributes(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$java_net_JarURLConnection$getMainAttributes(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$java_net_JarURLConnection$getCertificates(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getJarFile(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getJarEntry(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$connect(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getInputStream(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getContentLength(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getContentLengthLong(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getContent(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getContentType(Class callerClass, java.net.JarURLConnection that) { + checkJarURLAccess(callerClass, that); + } + + @Override + public void check$sun_net_www_protocol_jar_JarURLConnection$getHeaderField( + Class callerClass, + java.net.JarURLConnection that, + String name + ) { + checkJarURLAccess(callerClass, that); } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java index 5afffc84f77a8..8366bd7448544 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java @@ -9,12 +9,10 @@ package org.elasticsearch.entitlement.runtime.api; -public class NotEntitledException extends RuntimeException { +import java.security.AccessControlException; + +public class NotEntitledException extends AccessControlException { public NotEntitledException(String message) { super(message); } - - public NotEntitledException(String message, Throwable cause) { - super(message, cause); - } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java index 768babdb840f5..fef7b5d11aff0 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java @@ -9,6 +9,8 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -20,7 +22,7 @@ * using this annotation is considered parseable as part of a policy file * for entitlements. */ -@Target(ElementType.CONSTRUCTOR) +@Target({ ElementType.CONSTRUCTOR, ElementType.METHOD }) @Retention(RetentionPolicy.RUNTIME) public @interface ExternalEntitlement { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java index 55813df28b6f8..0e35ef0f0c72e 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java @@ -9,73 +9,256 @@ package org.elasticsearch.entitlement.runtime.policy; -import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; -import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; -final class FileAccessTree { - static final FileAccessTree EMPTY = new FileAccessTree(List.of()); +import static java.util.Comparator.comparing; +import static org.elasticsearch.core.PathUtils.getDefaultFileSystem; +import static org.elasticsearch.entitlement.runtime.policy.FileUtils.PATH_ORDER; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; +public final class FileAccessTree { + + /** + * An intermediary structure to help build exclusive paths for files entitlements. + */ + record ExclusiveFileEntitlement(String componentName, String moduleName, FilesEntitlement filesEntitlement) {} + + /** + * An intermediary structure to help globally validate exclusive paths, and then build exclusive paths for individual modules. + */ + record ExclusivePath(String componentName, Set moduleNames, String path) { + + @Override + public String toString() { + return "[[" + componentName + "] " + moduleNames + " [" + path + "]]"; + } + } + + static List buildExclusivePathList(List exclusiveFileEntitlements, PathLookup pathLookup) { + Map exclusivePaths = new HashMap<>(); + for (ExclusiveFileEntitlement efe : exclusiveFileEntitlements) { + for (FilesEntitlement.FileData fd : efe.filesEntitlement().filesData()) { + if (fd.exclusive()) { + List paths = fd.resolvePaths(pathLookup).toList(); + for (Path path : paths) { + String normalizedPath = normalizePath(path); + var exclusivePath = exclusivePaths.computeIfAbsent( + normalizedPath, + k -> new ExclusivePath(efe.componentName(), new HashSet<>(), normalizedPath) + ); + if (exclusivePath.componentName().equals(efe.componentName()) == false) { + throw new IllegalArgumentException( + "Path [" + + normalizedPath + + "] is already exclusive to [" + + exclusivePath.componentName() + + "]" + + exclusivePath.moduleNames + + ", cannot add exclusive access for [" + + efe.componentName() + + "][" + + efe.moduleName + + "]" + ); + } + exclusivePath.moduleNames.add(efe.moduleName()); + } + } + } + } + return exclusivePaths.values().stream().sorted(comparing(ExclusivePath::path, PATH_ORDER)).distinct().toList(); + } + + static void validateExclusivePaths(List exclusivePaths) { + if (exclusivePaths.isEmpty() == false) { + ExclusivePath currentExclusivePath = exclusivePaths.get(0); + for (int i = 1; i < exclusivePaths.size(); ++i) { + ExclusivePath nextPath = exclusivePaths.get(i); + if (currentExclusivePath.path().equals(nextPath.path) || isParent(currentExclusivePath.path(), nextPath.path())) { + throw new IllegalArgumentException( + "duplicate/overlapping exclusive paths found in files entitlements: " + currentExclusivePath + " and " + nextPath + ); + } + currentExclusivePath = nextPath; + } + } + } + + private static final Logger logger = LogManager.getLogger(FileAccessTree.class); + private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator(); + + private final String[] exclusivePaths; private final String[] readPaths; private final String[] writePaths; - FileAccessTree(List fileEntitlements) { + private FileAccessTree( + String componentName, + String moduleName, + FilesEntitlement filesEntitlement, + PathLookup pathLookup, + Path componentPath, + List exclusivePaths + ) { + List updatedExclusivePaths = new ArrayList<>(); + for (ExclusivePath exclusivePath : exclusivePaths) { + if (exclusivePath.componentName().equals(componentName) == false || exclusivePath.moduleNames().contains(moduleName) == false) { + updatedExclusivePaths.add(exclusivePath.path()); + } + } + List readPaths = new ArrayList<>(); List writePaths = new ArrayList<>(); - for (FileEntitlement fileEntitlement : fileEntitlements) { - var mode = fileEntitlement.mode(); - if (mode == FileEntitlement.Mode.READ_WRITE) { - writePaths.add(fileEntitlement.path()); + BiConsumer addPath = (path, mode) -> { + var normalized = normalizePath(path); + if (mode == READ_WRITE) { + writePaths.add(normalized); } - readPaths.add(fileEntitlement.path()); + readPaths.add(normalized); + }; + BiConsumer addPathAndMaybeLink = (path, mode) -> { + addPath.accept(path, mode); + // also try to follow symlinks. Lucene does this and writes to the target path. + if (Files.exists(path)) { + try { + Path realPath = path.toRealPath(); + if (realPath.equals(path) == false) { + addPath.accept(realPath, mode); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }; + for (FilesEntitlement.FileData fileData : filesEntitlement.filesData()) { + var platform = fileData.platform(); + if (platform != null && platform.isCurrent() == false) { + continue; + } + var mode = fileData.mode(); + var paths = fileData.resolvePaths(pathLookup); + paths.forEach(path -> { + if (path == null) { + // TODO: null paths shouldn't be allowed, but they can occur due to repo paths + return; + } + addPathAndMaybeLink.accept(path, mode); + }); } - readPaths.sort(String::compareTo); - writePaths.sort(String::compareTo); + // everything has access to the temp dir, config dir, to their own dir (their own jar files) and the jdk + addPathAndMaybeLink.accept(pathLookup.tempDir(), READ_WRITE); + // TODO: this grants read access to the config dir for all modules until explicit read entitlements can be added + addPathAndMaybeLink.accept(pathLookup.configDir(), Mode.READ); + if (componentPath != null) { + addPathAndMaybeLink.accept(componentPath, Mode.READ); + } + + // TODO: watcher uses javax.activation which looks for known mime types configuration, should this be global or explicit in watcher? + Path jdk = Paths.get(System.getProperty("java.home")); + addPathAndMaybeLink.accept(jdk.resolve("conf"), Mode.READ); + + updatedExclusivePaths.sort(PATH_ORDER); + readPaths.sort(PATH_ORDER); + writePaths.sort(PATH_ORDER); - this.readPaths = readPaths.toArray(new String[0]); - this.writePaths = writePaths.toArray(new String[0]); + this.exclusivePaths = updatedExclusivePaths.toArray(new String[0]); + this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]); + this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]); } - boolean canRead(Path path) { - return checkPath(normalize(path), readPaths); + // package private for testing + static List pruneSortedPaths(List paths) { + List prunedReadPaths = new ArrayList<>(); + if (paths.isEmpty() == false) { + String currentPath = paths.get(0); + prunedReadPaths.add(currentPath); + for (int i = 1; i < paths.size(); ++i) { + String nextPath = paths.get(i); + if (currentPath.equals(nextPath) == false && isParent(currentPath, nextPath) == false) { + prunedReadPaths.add(nextPath); + currentPath = nextPath; + } + } + } + return prunedReadPaths; } - @SuppressForbidden(reason = "Explicitly checking File apis") - boolean canRead(File file) { - return checkPath(normalize(file.toPath()), readPaths); + public static FileAccessTree of( + String componentName, + String moduleName, + FilesEntitlement filesEntitlement, + PathLookup pathLookup, + @Nullable Path componentPath, + List exclusivePaths + ) { + return new FileAccessTree(componentName, moduleName, filesEntitlement, pathLookup, componentPath, exclusivePaths); } - boolean canWrite(Path path) { - return checkPath(normalize(path), writePaths); + boolean canRead(Path path) { + return checkPath(normalizePath(path), readPaths); } - @SuppressForbidden(reason = "Explicitly checking File apis") - boolean canWrite(File file) { - return checkPath(normalize(file.toPath()), writePaths); + boolean canWrite(Path path) { + return checkPath(normalizePath(path), writePaths); } - private static String normalize(Path path) { - return path.toAbsolutePath().normalize().toString(); + /** + * @return the "canonical" form of the given {@code path}, to be used for entitlement checks. + */ + static String normalizePath(Path path) { + // Note that toAbsolutePath produces paths separated by the default file separator, + // so on Windows, if the given path uses forward slashes, this consistently + // converts it to backslashes. + String result = path.toAbsolutePath().normalize().toString(); + while (result.endsWith(FILE_SEPARATOR)) { + result = result.substring(0, result.length() - FILE_SEPARATOR.length()); + } + return result; } - private static boolean checkPath(String path, String[] paths) { + private boolean checkPath(String path, String[] paths) { + logger.trace(() -> Strings.format("checking [%s] against [%s]", path, String.join(",", paths))); if (paths.length == 0) { return false; } - int ndx = Arrays.binarySearch(paths, path); + + int endx = Arrays.binarySearch(exclusivePaths, path, PATH_ORDER); + if (endx < -1 && isParent(exclusivePaths[-endx - 2], path) || endx >= 0) { + return false; + } + + int ndx = Arrays.binarySearch(paths, path, PATH_ORDER); if (ndx < -1) { - String maybeParent = paths[-ndx - 2]; - return path.startsWith(maybeParent); + return isParent(paths[-ndx - 2], path); } return ndx >= 0; } + private static boolean isParent(String maybeParent, String path) { + logger.trace(() -> Strings.format("checking isParent [%s] for [%s]", maybeParent, path)); + return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length()); + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java deleted file mode 100644 index 4bd1dc10c85bb..0000000000000 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.entitlement.runtime.policy; - -import java.nio.file.Paths; - -/** - * Describes a file entitlement with a path and mode. - */ -public record FileEntitlement(String path, Mode mode) implements Entitlement { - - public enum Mode { - READ, - READ_WRITE - } - - public FileEntitlement { - path = normalizePath(path); - } - - private static String normalizePath(String path) { - return Paths.get(path).toAbsolutePath().normalize().toString(); - } - - private static Mode parseMode(String mode) { - if (mode.equals("read")) { - return Mode.READ; - } else if (mode.equals("read_write")) { - return Mode.READ_WRITE; - } else { - throw new PolicyValidationException("invalid mode: " + mode + ", valid values: [read, read_write]"); - } - } - - @ExternalEntitlement(parameterNames = { "path", "mode" }, esModulesOnly = false) - public FileEntitlement(String path, String mode) { - this(path, parseMode(mode)); - } -} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileUtils.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileUtils.java new file mode 100644 index 0000000000000..51caacc9d48b3 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileUtils.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.SuppressForbidden; + +import java.io.File; +import java.util.Comparator; + +import static java.lang.Character.isLetter; + +public class FileUtils { + + private FileUtils() {} + + /** + * For our lexicographic sort trick to work correctly, we must have path separators sort before + * any other character so that files in a directory appear immediately after that directory. + * For example, we require [/a, /a/b, /a.xml] rather than the natural order [/a, /a.xml, /a/b]. + */ + static final Comparator PATH_ORDER = (s1, s2) -> { + int len1 = s1.length(); + int len2 = s2.length(); + int lim = Math.min(len1, len2); + for (int k = 0; k < lim; k++) { + char c1 = s1.charAt(k); + char c2 = s2.charAt(k); + if (c1 == c2) { + continue; + } + boolean c1IsSeparator = isPathSeparator(c1); + boolean c2IsSeparator = isPathSeparator(c2); + if (c1IsSeparator == false || c2IsSeparator == false) { + if (c1IsSeparator) { + return -1; + } + if (c2IsSeparator) { + return 1; + } + return c1 - c2; + } + } + return len1 - len2; + }; + + @SuppressForbidden(reason = "we need the separator as a char, not a string") + private static boolean isPathSeparator(char c) { + return c == File.separatorChar; + } + + /** + * Tests if a path is absolute or relative, taking into consideration both Unix and Windows conventions. + * Note that this leads to a conflict, resolved in favor of Unix rules: `/foo` can be either a Unix absolute path, or a Windows + * relative path with "wrong" directory separator (using non-canonical / in Windows). + * This method is intended to be used as validation for different file entitlements format: therefore it is preferable to reject a + * relative path that is definitely absolute on Unix, rather than accept it as a possible relative path on Windows (if that is the case, + * the developer can easily fix the path by using the correct platform separators). + */ + public static boolean isAbsolutePath(String path) { + if (path.isEmpty()) { + return false; + } + if (path.charAt(0) == '/') { + // Unix/BSD absolute + return true; + } + + return isWindowsAbsolutePath(path); + } + + /** + * When testing for path separators in a platform-agnostic way, we may encounter both kinds of slashes, especially when + * processing windows paths. The JDK parses paths the same way under Windows. + */ + static boolean isSlash(char c) { + return (c == '\\') || (c == '/'); + } + + private static boolean isWindowsAbsolutePath(String input) { + // if a prefix is present, we expected (long) UNC or (long) absolute + if (input.startsWith("\\\\?\\")) { + return true; + } + + if (input.length() > 1) { + char c0 = input.charAt(0); + char c1 = input.charAt(1); + if (isSlash(c0) && isSlash(c1)) { + // Two slashes or more: UNC + return true; + } + if (isLetter(c0) && c1 == ':') { + // A drive: absolute + return true; + } + } + // Otherwise relative + return false; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java new file mode 100644 index 0000000000000..5cbe6108e009c --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.nio.file.Path; +import java.util.function.Function; +import java.util.stream.Stream; + +public record PathLookup( + Path homeDir, + Path configDir, + Path[] dataDirs, + Path[] sharedRepoDirs, + Path tempDir, + Function> settingResolver +) {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Platform.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Platform.java new file mode 100644 index 0000000000000..a209549782d0d --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Platform.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +public enum Platform { + LINUX, + MACOS, + WINDOWS; + + private static final Platform current = findCurrent(); + + private static Platform findCurrent() { + String os = System.getProperty("os.name"); + if (os.startsWith("Linux")) { + return LINUX; + } else if (os.startsWith("Mac OS")) { + return MACOS; + } else if (os.startsWith("Windows")) { + return WINDOWS; + } else { + throw new AssertionError("Unsupported platform [" + os + "]"); + } + } + + public boolean isCurrent() { + return this == current; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 04942e15d10a4..75e098a95902d 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -9,17 +9,37 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.Strings; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.instrumentation.InstrumentationService; import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusiveFileEntitlement; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusivePath; +import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import java.io.File; +import java.io.IOException; import java.lang.StackWalker.StackFrame; import java.lang.module.ModuleFinder; import java.lang.module.ModuleReference; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -35,28 +55,39 @@ import static java.util.function.Predicate.not; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toUnmodifiableMap; +import static java.util.zip.ZipFile.OPEN_DELETE; +import static java.util.zip.ZipFile.OPEN_READ; +import static org.elasticsearch.entitlement.bridge.Util.NO_CLASS; public class PolicyManager { - private static final Logger logger = LogManager.getLogger(PolicyManager.class); + /** + * Use this if you don't have a {@link ModuleEntitlements} in hand. + */ + private static final Logger generalLogger = LogManager.getLogger(PolicyManager.class); + + static final String UNKNOWN_COMPONENT_NAME = "(unknown)"; + static final String SERVER_COMPONENT_NAME = "(server)"; + static final String APM_AGENT_COMPONENT_NAME = "(APM agent)"; - record ModuleEntitlements(Map, List> entitlementsByType, FileAccessTree fileAccess) { - public static final ModuleEntitlements NONE = new ModuleEntitlements(Map.of(), FileAccessTree.EMPTY); + static final Class DEFAULT_FILESYSTEM_CLASS = PathUtils.getDefaultFileSystem().getClass(); + + static final Set MODULES_EXCLUDED_FROM_SYSTEM_MODULES = Set.of("java.desktop"); + + /** + * @param componentName the plugin name; or else one of the special component names + * like {@link #SERVER_COMPONENT_NAME} or {@link #APM_AGENT_COMPONENT_NAME}. + */ + record ModuleEntitlements( + String componentName, + Map, List> entitlementsByType, + FileAccessTree fileAccess, + Logger logger + ) { ModuleEntitlements { entitlementsByType = Map.copyOf(entitlementsByType); } - public static ModuleEntitlements from(List entitlements) { - var fileEntitlements = entitlements.stream() - .filter(e -> e.getClass().equals(FileEntitlement.class)) - .map(e -> (FileEntitlement) e) - .toList(); - return new ModuleEntitlements( - entitlements.stream().collect(groupingBy(Entitlement::getClass)), - new FileAccessTree(fileEntitlements) - ); - } - public boolean hasEntitlement(Class entitlementClass) { return entitlementsByType.containsKey(entitlementClass); } @@ -70,80 +101,166 @@ public Stream getEntitlements(Class entitlementCla } } + private FileAccessTree getDefaultFileAccess(String componentName, Path componentPath) { + return FileAccessTree.of(componentName, UNKNOWN_COMPONENT_NAME, FilesEntitlement.EMPTY, pathLookup, componentPath, List.of()); + } + + // pkg private for testing + ModuleEntitlements defaultEntitlements(String componentName, Path componentPath, String moduleName) { + return new ModuleEntitlements( + componentName, + Map.of(), + getDefaultFileAccess(componentName, componentPath), + getLogger(componentName, moduleName) + ); + } + + // pkg private for testing + ModuleEntitlements policyEntitlements(String componentName, Path componentPath, String moduleName, List entitlements) { + FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY; + for (Entitlement entitlement : entitlements) { + if (entitlement instanceof FilesEntitlement) { + filesEntitlement = (FilesEntitlement) entitlement; + } + } + return new ModuleEntitlements( + componentName, + entitlements.stream().collect(groupingBy(Entitlement::getClass)), + FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPath, exclusivePaths), + getLogger(componentName, moduleName) + ); + } + final Map moduleEntitlementsMap = new ConcurrentHashMap<>(); - protected final Map> serverEntitlements; - protected final List agentEntitlements; - protected final Map>> pluginsEntitlements; + private final Map> serverEntitlements; + private final List apmAgentEntitlements; + private final Map>> pluginsEntitlements; private final Function, String> pluginResolver; + private final PathLookup pathLookup; + private final Set> mutedClasses; public static final String ALL_UNNAMED = "ALL-UNNAMED"; - private static final Set systemModules = findSystemModules(); + private static final Set SYSTEM_LAYER_MODULES = findSystemLayerModules(); - private static Set findSystemModules() { + private static Set findSystemLayerModules() { var systemModulesDescriptors = ModuleFinder.ofSystem() .findAll() .stream() .map(ModuleReference::descriptor) .collect(Collectors.toUnmodifiableSet()); - return ModuleLayer.boot() - .modules() - .stream() - .filter(m -> systemModulesDescriptors.contains(m.getDescriptor())) - .collect(Collectors.toUnmodifiableSet()); + return Stream.concat( + // entitlements is a "system" module, we can do anything from it + Stream.of(PolicyManager.class.getModule()), + // anything in the boot layer is also part of the system + ModuleLayer.boot() + .modules() + .stream() + .filter( + m -> systemModulesDescriptors.contains(m.getDescriptor()) + && MODULES_EXCLUDED_FROM_SYSTEM_MODULES.contains(m.getName()) == false + ) + ).collect(Collectors.toUnmodifiableSet()); } + // Anything in the boot layer that is not in the system layer, is in the server layer + public static final Set SERVER_LAYER_MODULES = ModuleLayer.boot() + .modules() + .stream() + .filter(m -> SYSTEM_LAYER_MODULES.contains(m) == false) + .collect(Collectors.toUnmodifiableSet()); + + private final Map sourcePaths; /** - * The package name containing agent classes. + * The package name containing classes from the APM agent. */ - private final String agentsPackageName; + private final String apmAgentPackageName; /** * Frames originating from this module are ignored in the permission logic. */ private final Module entitlementsModule; + /** + * Paths that are only allowed for a single module. Used to generate + * structures to indicate other modules aren't allowed to use these + * files in {@link FileAccessTree}s. + */ + private final List exclusivePaths; + public PolicyManager( Policy serverPolicy, - List agentEntitlements, + List apmAgentEntitlements, Map pluginPolicies, Function, String> pluginResolver, - String agentsPackageName, - Module entitlementsModule + Map sourcePaths, + String apmAgentPackageName, + Module entitlementsModule, + PathLookup pathLookup, + Set> suppressFailureLogClasses ) { this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy)); - this.agentEntitlements = agentEntitlements; + this.apmAgentEntitlements = apmAgentEntitlements; this.pluginsEntitlements = requireNonNull(pluginPolicies).entrySet() .stream() .collect(toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue()))); this.pluginResolver = pluginResolver; - this.agentsPackageName = agentsPackageName; + this.sourcePaths = sourcePaths; + this.apmAgentPackageName = apmAgentPackageName; this.entitlementsModule = entitlementsModule; + this.pathLookup = requireNonNull(pathLookup); + this.mutedClasses = suppressFailureLogClasses; + + List exclusiveFileEntitlements = new ArrayList<>(); + for (var e : serverEntitlements.entrySet()) { + validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue(), exclusiveFileEntitlements); + } + validateEntitlementsPerModule(APM_AGENT_COMPONENT_NAME, ALL_UNNAMED, apmAgentEntitlements, exclusiveFileEntitlements); + for (var p : pluginsEntitlements.entrySet()) { + for (var m : p.getValue().entrySet()) { + validateEntitlementsPerModule(p.getKey(), m.getKey(), m.getValue(), exclusiveFileEntitlements); + } + } + List exclusivePaths = FileAccessTree.buildExclusivePathList(exclusiveFileEntitlements, pathLookup); + FileAccessTree.validateExclusivePaths(exclusivePaths); + this.exclusivePaths = exclusivePaths; } private static Map> buildScopeEntitlementsMap(Policy policy) { return policy.scopes().stream().collect(toUnmodifiableMap(Scope::moduleName, Scope::entitlements)); } + private static void validateEntitlementsPerModule( + String componentName, + String moduleName, + List entitlements, + List exclusiveFileEntitlements + ) { + Set> found = new HashSet<>(); + for (var e : entitlements) { + if (found.contains(e.getClass())) { + throw new IllegalArgumentException( + "[" + componentName + "] using module [" + moduleName + "] found duplicate entitlement [" + e.getClass().getName() + "]" + ); + } + found.add(e.getClass()); + if (e instanceof FilesEntitlement fe) { + exclusiveFileEntitlements.add(new ExclusiveFileEntitlement(componentName, moduleName, fe)); + } + } + } + public void checkStartProcess(Class callerClass) { - neverEntitled(callerClass, "start process"); + neverEntitled(callerClass, () -> "start process"); } - private void neverEntitled(Class callerClass, String operationDescription) { - var requestingClass = requestingClass(callerClass); - if (isTriviallyAllowed(requestingClass)) { - return; - } + public void checkWriteStoreAttributes(Class callerClass) { + neverEntitled(callerClass, () -> "change file store attributes"); + } - throw new NotEntitledException( - Strings.format( - "Not entitled: caller [%s], module [%s], operation [%s]", - callerClass, - requestingClass.getModule() == null ? "" : requestingClass.getModule().getName(), - operationDescription - ) - ); + public void checkReadStoreAttributes(Class callerClass) { + checkEntitlementPresent(callerClass, ReadStoreAttributesEntitlement.class); } /** @@ -156,13 +273,17 @@ private void neverEntitled(Class callerClass, Supplier operationDescr return; } - throw new NotEntitledException( + ModuleEntitlements entitlements = getEntitlements(requestingClass); + notEntitled( Strings.format( - "Not entitled: caller [%s], module [%s], operation [%s]", - callerClass, - requestingClass.getModule() == null ? "" : requestingClass.getModule().getName(), + "component [%s], module [%s], class [%s], operation [%s]", + entitlements.componentName(), + getModuleName(requestingClass), + requestingClass, operationDescription.get() - ) + ), + callerClass, + entitlements ); } @@ -179,17 +300,23 @@ public void checkSetHttpsConnectionProperties(Class callerClass) { } public void checkChangeJVMGlobalState(Class callerClass) { - neverEntitled(callerClass, () -> { - // Look up the check$ method to compose an informative error message. - // This way, we don't need to painstakingly describe every individual global-state change. - Optional checkMethodName = StackWalker.getInstance() - .walk( - frames -> frames.map(StackFrame::getMethodName) - .dropWhile(not(methodName -> methodName.startsWith("check$"))) - .findFirst() - ); - return checkMethodName.map(this::operationDescription).orElse("change JVM global state"); - }); + neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("change JVM global state")); + } + + public void checkLoggingFileHandler(Class callerClass) { + neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("create logging file handler")); + } + + private Optional walkStackForCheckMethodName() { + // Look up the check$ method to compose an informative error message. + // This way, we don't need to painstakingly describe every individual global-state change. + return StackWalker.getInstance() + .walk( + frames -> frames.map(StackFrame::getMethodName) + .dropWhile(not(methodName -> methodName.startsWith(InstrumentationService.CHECK_METHOD_PREFIX))) + .findFirst() + ) + .map(this::operationDescription); } /** @@ -200,72 +327,94 @@ public void checkChangeNetworkHandling(Class callerClass) { } /** - * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions + * Check for operations that can modify the way file operations are handled */ - public void checkReadSensitiveNetworkInformation(Class callerClass) { - neverEntitled(callerClass, "access sensitive network information"); + public void checkChangeFilesHandling(Class callerClass) { + checkChangeJVMGlobalState(callerClass); } @SuppressForbidden(reason = "Explicitly checking File apis") public void checkFileRead(Class callerClass, File file) { - var requestingClass = requestingClass(callerClass); - if (isTriviallyAllowed(requestingClass)) { - return; - } + checkFileRead(callerClass, file.toPath()); + } - ModuleEntitlements entitlements = getEntitlements(requestingClass); - if (entitlements.fileAccess().canRead(file) == false) { - throw new NotEntitledException( - Strings.format( - "Not entitled: caller [%s], module [%s], entitlement [file], operation [read], path [%s]", - callerClass, - requestingClass.getModule(), - file + private static boolean isPathOnDefaultFilesystem(Path path) { + var pathFileSystemClass = path.getFileSystem().getClass(); + if (path.getFileSystem().getClass() != DEFAULT_FILESYSTEM_CLASS) { + generalLogger.trace( + () -> Strings.format( + "File entitlement trivially allowed: path [%s] is for a different FileSystem class [%s], default is [%s]", + path.toString(), + pathFileSystemClass.getName(), + DEFAULT_FILESYSTEM_CLASS.getName() ) ); + return false; } + return true; } public void checkFileRead(Class callerClass, Path path) { + try { + checkFileRead(callerClass, path, false); + } catch (NoSuchFileException e) { + assert false : "NoSuchFileException should only be thrown when following links"; + var notEntitledException = new NotEntitledException(e.getMessage()); + notEntitledException.addSuppressed(e); + throw notEntitledException; + } + } + + public void checkFileRead(Class callerClass, Path path, boolean followLinks) throws NoSuchFileException { + if (isPathOnDefaultFilesystem(path) == false) { + return; + } var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { return; } ModuleEntitlements entitlements = getEntitlements(requestingClass); - if (entitlements.fileAccess().canRead(path) == false) { - throw new NotEntitledException( + + Path realPath = null; + boolean canRead = entitlements.fileAccess().canRead(path); + if (canRead && followLinks) { + try { + realPath = path.toRealPath(); + if (realPath.equals(path) == false) { + canRead = entitlements.fileAccess().canRead(realPath); + } + } catch (NoSuchFileException e) { + throw e; // rethrow + } catch (IOException e) { + canRead = false; + } + } + + if (canRead == false) { + notEntitled( Strings.format( - "Not entitled: caller [%s], module [%s], entitlement [file], operation [read], path [%s]", - callerClass, - requestingClass.getModule(), - path - ) + "component [%s], module [%s], class [%s], entitlement [file], operation [read], path [%s]", + entitlements.componentName(), + getModuleName(requestingClass), + requestingClass, + realPath == null ? path : Strings.format("%s -> %s", path, realPath) + ), + callerClass, + entitlements ); } } @SuppressForbidden(reason = "Explicitly checking File apis") public void checkFileWrite(Class callerClass, File file) { - var requestingClass = requestingClass(callerClass); - if (isTriviallyAllowed(requestingClass)) { - return; - } - - ModuleEntitlements entitlements = getEntitlements(requestingClass); - if (entitlements.fileAccess().canWrite(file) == false) { - throw new NotEntitledException( - Strings.format( - "Not entitled: caller [%s], module [%s], entitlement [file], operation [write], path [%s]", - callerClass, - requestingClass.getModule(), - file - ) - ); - } + checkFileWrite(callerClass, file.toPath()); } public void checkFileWrite(Class callerClass, Path path) { + if (isPathOnDefaultFilesystem(path) == false) { + return; + } var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { return; @@ -273,17 +422,53 @@ public void checkFileWrite(Class callerClass, Path path) { ModuleEntitlements entitlements = getEntitlements(requestingClass); if (entitlements.fileAccess().canWrite(path) == false) { - throw new NotEntitledException( + notEntitled( Strings.format( - "Not entitled: caller [%s], module [%s], entitlement [file], operation [write], path [%s]", - callerClass, - requestingClass.getModule(), + "component [%s], module [%s], class [%s], entitlement [file], operation [write], path [%s]", + entitlements.componentName(), + getModuleName(requestingClass), + requestingClass, path - ) + ), + callerClass, + entitlements ); } } + public void checkCreateTempFile(Class callerClass) { + checkFileWrite(callerClass, pathLookup.tempDir()); + } + + @SuppressForbidden(reason = "Explicitly checking File apis") + public void checkFileWithZipMode(Class callerClass, File file, int zipMode) { + assert zipMode == OPEN_READ || zipMode == (OPEN_READ | OPEN_DELETE); + if ((zipMode & OPEN_DELETE) == OPEN_DELETE) { + // This needs both read and write, but we happen to know that checkFileWrite + // actually checks both. + checkFileWrite(callerClass, file); + } else { + checkFileRead(callerClass, file); + } + } + + public void checkFileDescriptorRead(Class callerClass) { + neverEntitled(callerClass, () -> "read file descriptor"); + } + + public void checkFileDescriptorWrite(Class callerClass) { + neverEntitled(callerClass, () -> "write file descriptor"); + } + + /** + * Invoked when we try to get an arbitrary {@code FileAttributeView} class. Such a class can modify attributes, like owner etc.; + * we could think about introducing checks for each of the operations, but for now we over-approximate this and simply deny when it is + * used directly. + */ + public void checkGetFileAttributeView(Class callerClass) { + neverEntitled(callerClass, () -> "get file attribute view"); + } + /** * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions */ @@ -311,32 +496,43 @@ public void checkAllNetworkAccess(Class callerClass) { } var classEntitlements = getEntitlements(requestingClass); - if (classEntitlements.hasEntitlement(InboundNetworkEntitlement.class) == false) { - throw new NotEntitledException( + checkFlagEntitlement(classEntitlements, InboundNetworkEntitlement.class, requestingClass, callerClass); + checkFlagEntitlement(classEntitlements, OutboundNetworkEntitlement.class, requestingClass, callerClass); + } + + public void checkUnsupportedURLProtocolConnection(Class callerClass, String protocol) { + neverEntitled(callerClass, () -> Strings.format("unsupported URL protocol [%s]", protocol)); + } + + private void checkFlagEntitlement( + ModuleEntitlements classEntitlements, + Class entitlementClass, + Class requestingClass, + Class callerClass + ) { + if (classEntitlements.hasEntitlement(entitlementClass) == false) { + notEntitled( Strings.format( - "Missing entitlement: class [%s], module [%s], entitlement [inbound_network]", + "component [%s], module [%s], class [%s], entitlement [%s]", + classEntitlements.componentName(), + getModuleName(requestingClass), requestingClass, - requestingClass.getModule().getName() - ) + PolicyParser.buildEntitlementNameFromClass(entitlementClass) + ), + callerClass, + classEntitlements ); } - - if (classEntitlements.hasEntitlement(OutboundNetworkEntitlement.class) == false) { - throw new NotEntitledException( - Strings.format( - "Missing entitlement: class [%s], module [%s], entitlement [outbound_network]", + classEntitlements.logger() + .debug( + () -> Strings.format( + "Entitled: component [%s], module [%s], class [%s], entitlement [%s]", + classEntitlements.componentName(), + getModuleName(requestingClass), requestingClass, - requestingClass.getModule().getName() + PolicyParser.buildEntitlementNameFromClass(entitlementClass) ) ); - } - logger.debug( - () -> Strings.format( - "Entitled: class [%s], module [%s], entitlements [inbound_network, outbound_network]", - requestingClass, - requestingClass.getModule().getName() - ) - ); } public void checkWriteProperty(Class callerClass, String property) { @@ -347,52 +543,65 @@ public void checkWriteProperty(Class callerClass, String property) { ModuleEntitlements entitlements = getEntitlements(requestingClass); if (entitlements.getEntitlements(WriteSystemPropertiesEntitlement.class).anyMatch(e -> e.properties().contains(property))) { - logger.debug( - () -> Strings.format( - "Entitled: class [%s], module [%s], entitlement [write_system_properties], property [%s]", - requestingClass, - requestingClass.getModule().getName(), - property - ) - ); + entitlements.logger() + .debug( + () -> Strings.format( + "Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]", + entitlements.componentName(), + getModuleName(requestingClass), + requestingClass, + property + ) + ); return; } - throw new NotEntitledException( + notEntitled( Strings.format( - "Missing entitlement: class [%s], module [%s], entitlement [write_system_properties], property [%s]", + "component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]", + entitlements.componentName(), + getModuleName(requestingClass), requestingClass, - requestingClass.getModule().getName(), property - ) + ), + callerClass, + entitlements ); } + private void notEntitled(String message, Class callerClass, ModuleEntitlements entitlements) { + var exception = new NotEntitledException(message); + // Don't emit a log for muted classes, e.g. classes containing self tests + if (mutedClasses.contains(callerClass) == false) { + entitlements.logger().warn("Not entitled: {}", message, exception); + } + throw exception; + } + + private static Logger getLogger(String componentName, String moduleName) { + var loggerSuffix = "." + componentName + "." + ((moduleName == null) ? ALL_UNNAMED : moduleName); + return MODULE_LOGGERS.computeIfAbsent(PolicyManager.class.getName() + loggerSuffix, LogManager::getLogger); + } + + /** + * We want to use the same {@link Logger} object for a given name, because we want {@link ModuleEntitlements} + * {@code equals} and {@code hashCode} to work. + *

+ * This would not be required if LogManager + * memoized the loggers, + * but here we are. + */ + private static final ConcurrentHashMap MODULE_LOGGERS = new ConcurrentHashMap<>(); + + public void checkManageThreadsEntitlement(Class callerClass) { + checkEntitlementPresent(callerClass, ManageThreadsEntitlement.class); + } + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { return; } - - ModuleEntitlements entitlements = getEntitlements(requestingClass); - if (entitlements.hasEntitlement(entitlementClass)) { - logger.debug( - () -> Strings.format( - "Entitled: class [%s], module [%s], entitlement [%s]", - requestingClass, - requestingClass.getModule().getName(), - PolicyParser.getEntitlementTypeName(entitlementClass) - ) - ); - return; - } - throw new NotEntitledException( - Strings.format( - "Missing entitlement: class [%s], module [%s], entitlement [%s]", - requestingClass, - requestingClass.getModule().getName(), - PolicyParser.getEntitlementTypeName(entitlementClass) - ) - ); + checkFlagEntitlement(getEntitlements(requestingClass), entitlementClass, requestingClass, callerClass); } ModuleEntitlements getEntitlements(Class requestingClass) { @@ -402,45 +611,81 @@ ModuleEntitlements getEntitlements(Class requestingClass) { private ModuleEntitlements computeEntitlements(Class requestingClass) { Module requestingModule = requestingClass.getModule(); if (isServerModule(requestingModule)) { - return getModuleScopeEntitlements(requestingClass, serverEntitlements, requestingModule.getName(), "server"); + return getModuleScopeEntitlements( + serverEntitlements, + requestingModule.getName(), + SERVER_COMPONENT_NAME, + getComponentPathFromClass(requestingClass) + ); } // plugins var pluginName = pluginResolver.apply(requestingClass); if (pluginName != null) { var pluginEntitlements = pluginsEntitlements.get(pluginName); - if (pluginEntitlements != null) { - final String scopeName; - if (requestingModule.isNamed() == false) { - scopeName = ALL_UNNAMED; - } else { - scopeName = requestingModule.getName(); - } - return getModuleScopeEntitlements(requestingClass, pluginEntitlements, scopeName, pluginName); + if (pluginEntitlements == null) { + return defaultEntitlements(pluginName, sourcePaths.get(pluginName), requestingModule.getName()); + } else { + return getModuleScopeEntitlements( + pluginEntitlements, + getScopeName(requestingModule), + pluginName, + sourcePaths.get(pluginName) + ); } } - if (requestingModule.isNamed() == false && requestingClass.getPackageName().startsWith(agentsPackageName)) { - // agents are the only thing running non-modular in the system classloader - return ModuleEntitlements.from(agentEntitlements); + if (requestingModule.isNamed() == false && requestingClass.getPackageName().startsWith(apmAgentPackageName)) { + // The APM agent is the only thing running non-modular in the system classloader + return policyEntitlements( + APM_AGENT_COMPONENT_NAME, + getComponentPathFromClass(requestingClass), + ALL_UNNAMED, + apmAgentEntitlements + ); } - logger.warn("No applicable entitlement policy for class [{}]", requestingClass.getName()); - return ModuleEntitlements.NONE; + return defaultEntitlements(UNKNOWN_COMPONENT_NAME, null, requestingModule.getName()); + } + + private static String getScopeName(Module requestingModule) { + if (requestingModule.isNamed() == false) { + return ALL_UNNAMED; + } else { + return requestingModule.getName(); + } + } + + // pkg private for testing + static Path getComponentPathFromClass(Class requestingClass) { + var codeSource = requestingClass.getProtectionDomain().getCodeSource(); + if (codeSource == null) { + return null; + } + try { + return Paths.get(codeSource.getLocation().toURI()); + } catch (Exception e) { + // If we get a URISyntaxException, or any other Exception due to an invalid URI, we return null to safely skip this location + generalLogger.info( + "Cannot get component path for [{}]: [{}] cannot be converted to a valid Path", + requestingClass.getName(), + codeSource.getLocation().toString() + ); + return null; + } } private ModuleEntitlements getModuleScopeEntitlements( - Class callerClass, Map> scopeEntitlements, - String moduleName, - String component + String scopeName, + String componentName, + Path componentPath ) { - var entitlements = scopeEntitlements.get(moduleName); + var entitlements = scopeEntitlements.get(scopeName); if (entitlements == null) { - logger.warn("No applicable entitlement policy for [{}], module [{}], class [{}]", component, moduleName, callerClass); - return ModuleEntitlements.NONE; + return defaultEntitlements(componentName, componentPath, scopeName); } - return ModuleEntitlements.from(entitlements); + return policyEntitlements(componentName, componentPath, scopeName, entitlements); } private static boolean isServerModule(Module requestingModule) { @@ -462,19 +707,16 @@ Class requestingClass(Class callerClass) { return callerClass; } Optional> result = StackWalker.getInstance(RETAIN_CLASS_REFERENCE) - .walk(frames -> findRequestingClass(frames.map(StackFrame::getDeclaringClass))); + .walk(frames -> findRequestingFrame(frames).map(StackFrame::getDeclaringClass)); return result.orElse(null); } /** - * Given a stream of classes corresponding to the frames from a {@link StackWalker}, - * returns the module whose entitlements should be checked. - * - * @throws NullPointerException if the requesting module is {@code null} + * Given a stream of {@link StackFrame}s, identify the one whose entitlements should be checked. */ - Optional> findRequestingClass(Stream> classes) { - return classes.filter(c -> c.getModule() != entitlementsModule) // Ignore the entitlements library - .skip(1) // Skip the sensitive caller method + Optional findRequestingFrame(Stream frames) { + return frames.filter(f -> f.getDeclaringClass().getModule() != entitlementsModule) // ignore entitlements library + .skip(1) // Skip the sensitive caller method .findFirst(); } @@ -482,21 +724,33 @@ Optional> findRequestingClass(Stream> classes) { * @return true if permission is granted regardless of the entitlement */ private static boolean isTriviallyAllowed(Class requestingClass) { - if (logger.isTraceEnabled()) { - logger.trace("Stack trace for upcoming trivially-allowed check", new Exception()); + if (generalLogger.isTraceEnabled()) { + generalLogger.trace("Stack trace for upcoming trivially-allowed check", new Exception()); } if (requestingClass == null) { - logger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library"); + generalLogger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library"); + return true; + } + if (requestingClass == NO_CLASS) { + generalLogger.debug("Entitlement trivially allowed from outermost frame"); return true; } - if (systemModules.contains(requestingClass.getModule())) { - logger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName()); + if (SYSTEM_LAYER_MODULES.contains(requestingClass.getModule())) { + generalLogger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName()); return true; } - logger.trace("Entitlement not trivially allowed"); + generalLogger.trace("Entitlement not trivially allowed"); return false; } + /** + * @return the {@code requestingClass}'s module name as it would appear in an entitlement policy file + */ + private static String getModuleName(Class requestingClass) { + String name = requestingClass.getModule().getName(); + return (name == null) ? ALL_UNNAMED : name; + } + @Override public String toString() { return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}'; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index 95437027239b0..6ff86f3f30dbf 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -9,6 +9,16 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteAllSystemPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; import org.elasticsearch.xcontent.XContentLocation; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -19,12 +29,16 @@ import java.io.UncheckedIOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -35,22 +49,29 @@ */ public class PolicyParser { - private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( - FileEntitlement.class, + private static final Map> EXTERNAL_ENTITLEMENT_CLASSES_BY_NAME = Stream.of( CreateClassLoaderEntitlement.class, - SetHttpsConnectionPropertiesEntitlement.class, - OutboundNetworkEntitlement.class, + FilesEntitlement.class, InboundNetworkEntitlement.class, - WriteSystemPropertiesEntitlement.class, - LoadNativeLibrariesEntitlement.class - ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); + LoadNativeLibrariesEntitlement.class, + ManageThreadsEntitlement.class, + OutboundNetworkEntitlement.class, + SetHttpsConnectionPropertiesEntitlement.class, + WriteAllSystemPropertiesEntitlement.class, + WriteSystemPropertiesEntitlement.class + ).collect(Collectors.toUnmodifiableMap(PolicyParser::buildEntitlementNameFromClass, Function.identity())); + + private static final Map, String> EXTERNAL_ENTITLEMENT_NAMES_BY_CLASS = + EXTERNAL_ENTITLEMENT_CLASSES_BY_NAME.entrySet() + .stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getValue, Map.Entry::getKey)); protected final XContentParser policyParser; protected final String policyName; private final boolean isExternalPlugin; - private final Map> externalEntitlements; + private final Map> externalEntitlements; - static String getEntitlementTypeName(Class entitlementClass) { + static String buildEntitlementNameFromClass(Class entitlementClass) { var entitlementClassName = entitlementClass.getSimpleName(); if (entitlementClassName.endsWith("Entitlement") == false) { @@ -66,19 +87,79 @@ static String getEntitlementTypeName(Class entitlementCla .collect(Collectors.joining("_")); } + public static String getEntitlementName(Class entitlementClass) { + return EXTERNAL_ENTITLEMENT_NAMES_BY_CLASS.get(entitlementClass); + } + public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException { - this(inputStream, policyName, isExternalPlugin, EXTERNAL_ENTITLEMENTS); + this(inputStream, policyName, isExternalPlugin, EXTERNAL_ENTITLEMENT_CLASSES_BY_NAME); } // package private for tests - PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin, Map> externalEntitlements) - throws IOException { + PolicyParser( + InputStream inputStream, + String policyName, + boolean isExternalPlugin, + Map> externalEntitlements + ) throws IOException { this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); this.policyName = policyName; this.isExternalPlugin = isExternalPlugin; this.externalEntitlements = externalEntitlements; } + public VersionedPolicy parseVersionedPolicy() { + Set versions = Set.of(); + Policy policy = emptyPolicy(); + try { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException("expected object "); + } + + while (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + if (policyParser.currentToken() == XContentParser.Token.FIELD_NAME) { + if (policyParser.currentName().equals("versions")) { + versions = parseVersions(); + } else if (policyParser.currentName().equals("policy")) { + policy = parsePolicy(); + } else { + throw newPolicyParserException("expected either or field"); + } + } else { + throw newPolicyParserException("expected either or field"); + } + } + + return new VersionedPolicy(policy, versions); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + + private Policy emptyPolicy() { + return new Policy(policyName, List.of()); + } + + private Set parseVersions() throws IOException { + try { + if (policyParser.nextToken() != XContentParser.Token.START_ARRAY) { + throw newPolicyParserException("expected array of "); + } + Set versions = new HashSet<>(); + while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) { + if (policyParser.currentToken() == XContentParser.Token.VALUE_STRING) { + String version = policyParser.text(); + versions.add(version); + } else { + throw newPolicyParserException("expected "); + } + } + return versions; + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + public Policy parsePolicy() { try { if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { @@ -139,6 +220,7 @@ protected Entitlement parseEntitlement(String scopeName, String entitlementType) } Constructor entitlementConstructor = null; + Method entitlementMethod = null; ExternalEntitlement entitlementMetadata = null; for (var ctor : entitlementClass.getConstructors()) { var metadata = ctor.getAnnotation(ExternalEntitlement.class); @@ -153,8 +235,27 @@ protected Entitlement parseEntitlement(String scopeName, String entitlementType) entitlementConstructor = ctor; entitlementMetadata = metadata; } - } + for (var method : entitlementClass.getMethods()) { + var metadata = method.getAnnotation(ExternalEntitlement.class); + if (metadata != null) { + if (Modifier.isStatic(method.getModifiers()) == false) { + throw new IllegalStateException( + "entitlement class [" + entitlementClass.getName() + "] has non-static method annotated with ExternalEntitlement" + ); + } + if (entitlementMetadata != null) { + throw new IllegalStateException( + "entitlement class [" + + entitlementClass.getName() + + "] has more than one constructor and/or method annotated with ExternalEntitlement" + ); + } + entitlementMethod = method; + entitlementMetadata = metadata; + } + } + if (entitlementMetadata == null) { throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); } @@ -163,40 +264,53 @@ protected Entitlement parseEntitlement(String scopeName, String entitlementType) throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules"); } - Class[] parameterTypes = entitlementConstructor.getParameterTypes(); + Class[] parameterTypes = entitlementConstructor != null + ? entitlementConstructor.getParameterTypes() + : entitlementMethod.getParameterTypes(); String[] parametersNames = entitlementMetadata.parameterNames(); + Object[] parameterValues = new Object[parameterTypes.length]; if (parameterTypes.length != 0 || parametersNames.length != 0) { - if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { - throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); - } - } + if (policyParser.nextToken() == XContentParser.Token.START_OBJECT) { + Map parsedValues = policyParser.map(); - Map parsedValues = policyParser.map(); - - Object[] parameterValues = new Object[parameterTypes.length]; - for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) { - String parameterName = parametersNames[parameterIndex]; - Object parameterValue = parsedValues.remove(parameterName); - if (parameterValue == null) { - throw newPolicyParserException(scopeName, entitlementType, "missing entitlement parameter [" + parameterName + "]"); - } - Class parameterType = parameterTypes[parameterIndex]; - if (parameterType.isAssignableFrom(parameterValue.getClass()) == false) { - throw newPolicyParserException( - scopeName, - entitlementType, - "unexpected parameter type [" + parameterType.getSimpleName() + "] for entitlement parameter [" + parameterName + "]" - ); + for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) { + String parameterName = parametersNames[parameterIndex]; + Object parameterValue = parsedValues.remove(parameterName); + if (parameterValue == null) { + throw newPolicyParserException(scopeName, entitlementType, "missing entitlement parameter [" + parameterName + "]"); + } + Class parameterType = parameterTypes[parameterIndex]; + if (parameterType.isAssignableFrom(parameterValue.getClass()) == false) { + throw newPolicyParserException( + scopeName, + entitlementType, + "unexpected parameter type [" + + parameterType.getSimpleName() + + "] for entitlement parameter [" + + parameterName + + "]" + ); + } + parameterValues[parameterIndex] = parameterValue; + } + if (parsedValues.isEmpty() == false) { + throw newPolicyParserException(scopeName, entitlementType, "extraneous entitlement parameter(s) " + parsedValues); + } + } else if (policyParser.currentToken() == XContentParser.Token.START_ARRAY) { + List parsedValues = policyParser.list(); + parameterValues[0] = parsedValues; + } else { + throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); } - parameterValues[parameterIndex] = parameterValue; - } - if (parsedValues.isEmpty() == false) { - throw newPolicyParserException(scopeName, entitlementType, "extraneous entitlement parameter(s) " + parsedValues); } try { - return (Entitlement) entitlementConstructor.newInstance(parameterValues); + if (entitlementConstructor != null) { + return (Entitlement) entitlementConstructor.newInstance(parameterValues); + } else { + return (Entitlement) entitlementMethod.invoke(null, parameterValues); + } } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { if (e.getCause() instanceof PolicyValidationException piae) { throw newPolicyParserException(startLocation, scopeName, entitlementType, piae); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserUtils.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserUtils.java deleted file mode 100644 index 6e1ea8551825b..0000000000000 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserUtils.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.entitlement.runtime.policy; - -import org.elasticsearch.core.Strings; - -import java.io.IOException; -import java.lang.module.ModuleFinder; -import java.lang.module.ModuleReference; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.Objects.requireNonNull; -import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; - -public class PolicyParserUtils { - - public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) { - public PluginData { - requireNonNull(pluginPath); - } - } - - private static final String POLICY_FILE_NAME = "entitlement-policy.yaml"; - - public static Map createPluginPolicies(Collection pluginData) throws IOException { - Map pluginPolicies = new HashMap<>(pluginData.size()); - for (var entry : pluginData) { - Path pluginRoot = entry.pluginPath(); - String pluginName = pluginRoot.getFileName().toString(); - - final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin()); - - pluginPolicies.put(pluginName, policy); - } - return pluginPolicies; - } - - private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin) - throws IOException { - Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME); - - final Set moduleNames = getModuleNames(pluginRoot, isModular); - final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin); - - // TODO: should this check actually be part of the parser? - for (Scope scope : policy.scopes()) { - if (moduleNames.contains(scope.moduleName()) == false) { - throw new IllegalStateException( - Strings.format( - "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy file [%s]", - pluginName, - scope.moduleName(), - String.join(", ", moduleNames), - policyFile - ) - ); - } - } - return policy; - } - - private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException { - if (Files.exists(policyFile)) { - return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy(); - } - return new Policy(pluginName, List.of()); - } - - private static Set getModuleNames(Path pluginRoot, boolean isModular) { - if (isModular) { - ModuleFinder moduleFinder = ModuleFinder.of(pluginRoot); - Set moduleReferences = moduleFinder.findAll(); - - return moduleReferences.stream().map(mr -> mr.descriptor().name()).collect(Collectors.toUnmodifiableSet()); - } - // When isModular == false we use the same "ALL-UNNAMED" constant as the JDK to indicate (any) unnamed module for this plugin - return Set.of(ALL_UNNAMED); - } - -} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java new file mode 100644 index 0000000000000..1fdc8c5ad4227 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java @@ -0,0 +1,267 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.Strings; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.module.ModuleFinder; +import java.lang.module.ModuleReference; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Objects.requireNonNull; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; + +public class PolicyUtils { + + private static final Logger logger = LogManager.getLogger(PolicyUtils.class); + + public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) { + public PluginData { + requireNonNull(pluginPath); + } + } + + public static final String POLICY_FILE_NAME = "entitlement-policy.yaml"; + + public static Map createPluginPolicies( + Collection pluginData, + Map pluginPolicyPatches, + String version + ) throws IOException { + Map pluginPolicies = new HashMap<>(pluginData.size()); + for (var entry : pluginData) { + Path pluginRoot = entry.pluginPath(); + String pluginName = pluginRoot.getFileName().toString(); + final Set moduleNames = getModuleNames(pluginRoot, entry.isModular()); + + var pluginPolicyPatch = parseEncodedPolicyIfExists( + pluginPolicyPatches.get(pluginName), + version, + entry.isExternalPlugin(), + pluginName, + moduleNames + ); + var pluginPolicy = parsePolicyIfExists(pluginName, pluginRoot, entry.isExternalPlugin()); + validatePolicyScopes(pluginName, pluginPolicy, moduleNames, pluginRoot.resolve(POLICY_FILE_NAME).toString()); + + pluginPolicies.put( + pluginName, + pluginPolicyPatch == null + ? pluginPolicy + : new Policy(pluginPolicy.name(), PolicyUtils.mergeScopes(pluginPolicy.scopes(), pluginPolicyPatch.scopes())) + ); + } + return pluginPolicies; + } + + public static Policy parseEncodedPolicyIfExists( + String encodedPolicy, + String version, + boolean externalPlugin, + String layerName, + Set moduleNames + ) { + if (encodedPolicy != null) { + try { + var versionedPolicy = decodeEncodedPolicy(encodedPolicy, layerName, externalPlugin); + validatePolicyScopes(layerName, versionedPolicy.policy(), moduleNames, ""); + + // Empty versions defaults to "any" + if (versionedPolicy.versions().isEmpty() || versionedPolicy.versions().contains(version)) { + logger.info("Using policy patch for layer [{}]", layerName); + return versionedPolicy.policy(); + } else { + logger.warn( + "Found a policy patch with version mismatch. The patch will not be applied. " + + "Layer [{}]; policy versions [{}]; current version [{}]", + layerName, + String.join(",", versionedPolicy.versions()), + version + ); + } + } catch (Exception ex) { + logger.warn( + Strings.format("Found a policy patch with invalid content. The patch will not be applied. Layer [%s]", layerName), + ex + ); + } + } + return null; + } + + static VersionedPolicy decodeEncodedPolicy(String base64String, String layerName, boolean isExternalPlugin) throws IOException { + byte[] policyDefinition = Base64.getDecoder().decode(base64String); + return new PolicyParser(new ByteArrayInputStream(policyDefinition), layerName, isExternalPlugin).parseVersionedPolicy(); + } + + private static void validatePolicyScopes(String layerName, Policy policy, Set moduleNames, String policyLocation) { + // TODO: should this check actually be part of the parser? + for (Scope scope : policy.scopes()) { + if (moduleNames.contains(scope.moduleName()) == false) { + throw new IllegalStateException( + Strings.format( + "Invalid module name in policy: layer [%s] does not have module [%s]; available modules [%s]; policy path [%s]", + layerName, + scope.moduleName(), + String.join(", ", moduleNames), + policyLocation + ) + ); + } + } + } + + public static Policy parsePolicyIfExists(String pluginName, Path pluginRoot, boolean isExternalPlugin) throws IOException { + Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME); + if (Files.exists(policyFile)) { + try (var inputStream = Files.newInputStream(policyFile, StandardOpenOption.READ)) { + return new PolicyParser(inputStream, pluginName, isExternalPlugin).parsePolicy(); + } + } + return new Policy(pluginName, List.of()); + } + + private static Set getModuleNames(Path pluginRoot, boolean isModular) { + if (isModular) { + ModuleFinder moduleFinder = ModuleFinder.of(pluginRoot); + Set moduleReferences = moduleFinder.findAll(); + + return moduleReferences.stream().map(mr -> mr.descriptor().name()).collect(Collectors.toUnmodifiableSet()); + } + // When isModular == false we use the same "ALL-UNNAMED" constant as the JDK to indicate (any) unnamed module for this plugin + return Set.of(ALL_UNNAMED); + } + + public static List mergeScopes(List mainScopes, List additionalScopes) { + var result = new ArrayList(); + var additionalScopesMap = additionalScopes.stream().collect(Collectors.toMap(Scope::moduleName, Scope::entitlements)); + for (var mainScope : mainScopes) { + List additionalEntitlements = additionalScopesMap.remove(mainScope.moduleName()); + if (additionalEntitlements == null) { + result.add(mainScope); + } else { + result.add(new Scope(mainScope.moduleName(), mergeEntitlements(mainScope.entitlements(), additionalEntitlements))); + } + } + + for (var remainingEntry : additionalScopesMap.entrySet()) { + result.add(new Scope(remainingEntry.getKey(), remainingEntry.getValue())); + } + return result; + } + + static List mergeEntitlements(List a, List b) { + Map, Entitlement> entitlementMap = a.stream() + .collect(Collectors.toMap(Entitlement::getClass, Function.identity())); + + for (var entitlement : b) { + entitlementMap.merge(entitlement.getClass(), entitlement, PolicyUtils::mergeEntitlement); + } + return entitlementMap.values().stream().toList(); + } + + static Entitlement mergeEntitlement(Entitlement entitlement, Entitlement other) { + if (entitlement instanceof FilesEntitlement e) { + return mergeFiles(Stream.of(e, (FilesEntitlement) other)); + } + if (entitlement instanceof WriteSystemPropertiesEntitlement e) { + return mergeWriteSystemProperties(Stream.of(e, (WriteSystemPropertiesEntitlement) other)); + } + return entitlement; + } + + public static List mergeEntitlements(Stream entitlements) { + Map, List> entitlementMap = entitlements.collect( + Collectors.groupingBy(Entitlement::getClass) + ); + + List result = new ArrayList<>(); + for (var kv : entitlementMap.entrySet()) { + var entitlementClass = kv.getKey(); + var classEntitlements = kv.getValue(); + if (classEntitlements.size() == 1) { + result.add(classEntitlements.get(0)); + } else { + result.add(PolicyUtils.mergeEntitlement(entitlementClass, classEntitlements.stream())); + } + } + return result; + } + + static Entitlement mergeEntitlement(Class entitlementClass, Stream entitlements) { + if (entitlementClass.equals(FilesEntitlement.class)) { + return mergeFiles(entitlements.map(FilesEntitlement.class::cast)); + } else if (entitlementClass.equals(WriteSystemPropertiesEntitlement.class)) { + return mergeWriteSystemProperties(entitlements.map(WriteSystemPropertiesEntitlement.class::cast)); + } + return entitlements.findFirst().orElseThrow(); + } + + private static FilesEntitlement mergeFiles(Stream entitlements) { + return new FilesEntitlement(entitlements.flatMap(x -> x.filesData().stream()).distinct().toList()); + } + + private static WriteSystemPropertiesEntitlement mergeWriteSystemProperties(Stream entitlements) { + return new WriteSystemPropertiesEntitlement( + entitlements.flatMap(x -> x.properties().stream()).collect(Collectors.toUnmodifiableSet()) + ); + } + + static Set describeEntitlement(Entitlement entitlement) { + Set descriptions = new HashSet<>(); + if (entitlement instanceof FilesEntitlement f) { + f.filesData() + .stream() + .filter(x -> x.platform() == null || x.platform().isCurrent()) + .map(x -> Strings.format("%s %s", PolicyParser.getEntitlementName(FilesEntitlement.class), x.description())) + .forEach(descriptions::add); + } else if (entitlement instanceof WriteSystemPropertiesEntitlement w) { + w.properties() + .stream() + .map(p -> Strings.format("%s [%s]", PolicyParser.getEntitlementName(WriteSystemPropertiesEntitlement.class), p)) + .forEach(descriptions::add); + } else { + descriptions.add(PolicyParser.getEntitlementName(entitlement.getClass())); + } + return descriptions; + } + + /** + * Extract a unique set of entitlements descriptions from the plugin's policy file. Each entitlement is formatted for output to users. + */ + public static Set getEntitlementsDescriptions(Policy pluginPolicy) { + var allEntitlements = PolicyUtils.mergeEntitlements(pluginPolicy.scopes().stream().flatMap(scope -> scope.entitlements().stream())); + Set descriptions = new HashSet<>(); + for (var entitlement : allEntitlements) { + descriptions.addAll(PolicyUtils.describeEntitlement(entitlement)); + } + return descriptions; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyValidationException.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyValidationException.java index a2bc49d99b44f..5f21db011884d 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyValidationException.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyValidationException.java @@ -15,13 +15,13 @@ * parser is able to wrap this exception with a line/character number for * additional useful error information. */ -class PolicyValidationException extends RuntimeException { +public class PolicyValidationException extends RuntimeException { - PolicyValidationException(String message) { + public PolicyValidationException(String message) { super(message); } - PolicyValidationException(String message, Throwable cause) { + public PolicyValidationException(String message, Throwable cause) { super(message, cause); } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java index 55e257797d603..6342a155da940 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java @@ -9,6 +9,8 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; + import java.util.List; import java.util.Objects; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/VersionedPolicy.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/VersionedPolicy.java new file mode 100644 index 0000000000000..5cb3effd62383 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/VersionedPolicy.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.Set; + +/** + * A Policy and associated versions to which the policy applies + */ +public record VersionedPolicy(Policy policy, Set versions) {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/CreateClassLoaderEntitlement.java similarity index 81% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/CreateClassLoaderEntitlement.java index 55e4b66595642..4b7137f8c7cd6 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/CreateClassLoaderEntitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; public record CreateClassLoaderEntitlement() implements Entitlement { @ExternalEntitlement diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/Entitlement.java similarity index 83% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/Entitlement.java index 5b53c399cc1b7..996b8a19ac8b0 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/Entitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.Policy; /** * Marker interface to ensure that only {@link Entitlement} are diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ExitVMEntitlement.java similarity index 90% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ExitVMEntitlement.java index e5c836ea22b20..470277c482461 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ExitVMEntitlement.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; /** * Internal policy type (not-parseable -- not available to plugins). diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java new file mode 100644 index 0000000000000..74afdb2e572f2 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.core.Strings; +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; +import org.elasticsearch.entitlement.runtime.policy.FileUtils; +import org.elasticsearch.entitlement.runtime.policy.PathLookup; +import org.elasticsearch.entitlement.runtime.policy.Platform; +import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException; + +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiFunction; +import java.util.stream.Stream; + +/** + * Describes a file entitlement with a path and mode. + */ +public record FilesEntitlement(List filesData) implements Entitlement { + + public static final String SEPARATOR = FileSystems.getDefault().getSeparator(); + + public static final FilesEntitlement EMPTY = new FilesEntitlement(List.of()); + + public enum Mode { + READ, + READ_WRITE + } + + public enum BaseDir { + CONFIG, + DATA, + SHARED_REPO, + HOME + } + + public sealed interface FileData { + + Stream resolvePaths(PathLookup pathLookup); + + Mode mode(); + + boolean exclusive(); + + FileData withExclusive(boolean exclusive); + + Platform platform(); + + FileData withPlatform(Platform platform); + + String description(); + + static FileData ofPath(Path path, Mode mode) { + return new AbsolutePathFileData(path, mode, null, false); + } + + static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) { + return new RelativePathFileData(relativePath, baseDir, mode, null, false); + } + + static FileData ofPathSetting(String setting, BaseDir baseDir, Mode mode) { + return new PathSettingFileData(setting, baseDir, mode, null, false); + } + } + + private sealed interface RelativeFileData extends FileData { + BaseDir baseDir(); + + Stream resolveRelativePaths(PathLookup pathLookup); + + @Override + default Stream resolvePaths(PathLookup pathLookup) { + Objects.requireNonNull(pathLookup); + var relativePaths = resolveRelativePaths(pathLookup); + switch (baseDir()) { + case CONFIG: + return relativePaths.map(relativePath -> pathLookup.configDir().resolve(relativePath)); + case DATA: + return relativePathsCombination(pathLookup.dataDirs(), relativePaths); + case SHARED_REPO: + return relativePathsCombination(pathLookup.sharedRepoDirs(), relativePaths); + case HOME: + return relativePaths.map(relativePath -> pathLookup.homeDir().resolve(relativePath)); + default: + throw new IllegalArgumentException(); + } + } + } + + private static Stream relativePathsCombination(Path[] baseDirs, Stream relativePaths) { + // multiple base dirs are a pain...we need the combination of the base dirs and relative paths + List paths = new ArrayList<>(); + for (var relativePath : relativePaths.toList()) { + for (var dataDir : baseDirs) { + paths.add(dataDir.resolve(relativePath)); + } + } + return paths.stream(); + } + + private record AbsolutePathFileData(Path path, Mode mode, Platform platform, boolean exclusive) implements FileData { + + @Override + public AbsolutePathFileData withExclusive(boolean exclusive) { + return new AbsolutePathFileData(path, mode, platform, exclusive); + } + + @Override + public Stream resolvePaths(PathLookup pathLookup) { + return Stream.of(path); + } + + @Override + public FileData withPlatform(Platform platform) { + if (platform == platform()) { + return this; + } + return new AbsolutePathFileData(path, mode, platform, exclusive); + } + + @Override + public String description() { + return Strings.format("[%s] %s%s", mode, path.toAbsolutePath().normalize(), exclusive ? " (exclusive)" : ""); + } + } + + private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode, Platform platform, boolean exclusive) + implements + FileData, + RelativeFileData { + + @Override + public RelativePathFileData withExclusive(boolean exclusive) { + return new RelativePathFileData(relativePath, baseDir, mode, platform, exclusive); + } + + @Override + public Stream resolveRelativePaths(PathLookup pathLookup) { + return Stream.of(relativePath); + } + + @Override + public FileData withPlatform(Platform platform) { + if (platform == platform()) { + return this; + } + return new RelativePathFileData(relativePath, baseDir, mode, platform, exclusive); + } + + @Override + public String description() { + return Strings.format("[%s] <%s>%s%s%s", mode, baseDir, SEPARATOR, relativePath, exclusive ? " (exclusive)" : ""); + } + } + + private record PathSettingFileData(String setting, BaseDir baseDir, Mode mode, Platform platform, boolean exclusive) + implements + RelativeFileData { + + @Override + public PathSettingFileData withExclusive(boolean exclusive) { + return new PathSettingFileData(setting, baseDir, mode, platform, exclusive); + } + + @Override + public Stream resolveRelativePaths(PathLookup pathLookup) { + Stream result = pathLookup.settingResolver() + .apply(setting) + .filter(s -> s.toLowerCase(Locale.ROOT).startsWith("https://") == false) + .distinct(); + return result.map(Path::of); + } + + @Override + public FileData withPlatform(Platform platform) { + if (platform == platform()) { + return this; + } + return new PathSettingFileData(setting, baseDir, mode, platform, exclusive); + } + + @Override + public String description() { + return Strings.format("[%s] <%s>%s<%s>%s", mode, baseDir, SEPARATOR, setting, exclusive ? " (exclusive)" : ""); + } + } + + private static Mode parseMode(String mode) { + if (mode.equals("read")) { + return Mode.READ; + } else if (mode.equals("read_write")) { + return Mode.READ_WRITE; + } else { + throw new PolicyValidationException("invalid mode: " + mode + ", valid values: [read, read_write]"); + } + } + + private static Platform parsePlatform(String platform) { + if (platform.equals("linux")) { + return Platform.LINUX; + } else if (platform.equals("macos")) { + return Platform.MACOS; + } else if (platform.equals("windows")) { + return Platform.WINDOWS; + } else { + throw new PolicyValidationException("invalid platform: " + platform + ", valid values: [linux, macos, windows]"); + } + } + + private static BaseDir parseBaseDir(String baseDir) { + return switch (baseDir) { + case "config" -> BaseDir.CONFIG; + case "data" -> BaseDir.DATA; + case "home" -> BaseDir.HOME; + // NOTE: shared_repo is _not_ accessible to policy files, only internally + default -> throw new PolicyValidationException( + "invalid relative directory: " + baseDir + ", valid values: [config, data, home]" + ); + }; + } + + @ExternalEntitlement(parameterNames = { "paths" }, esModulesOnly = false) + @SuppressWarnings("unchecked") + public static FilesEntitlement build(List paths) { + if (paths == null || paths.isEmpty()) { + throw new PolicyValidationException("must specify at least one path"); + } + BiFunction, String, String> checkString = (values, key) -> { + Object value = values.remove(key); + if (value == null) { + return null; + } else if (value instanceof String str) { + return str; + } + throw new PolicyValidationException( + "expected [" + + key + + "] to be type [" + + String.class.getSimpleName() + + "] but found type [" + + value.getClass().getSimpleName() + + "]" + ); + }; + BiFunction, String, Boolean> checkBoolean = (values, key) -> { + Object value = values.remove(key); + if (value == null) { + return null; + } else if (value instanceof Boolean bool) { + return bool; + } + throw new PolicyValidationException( + "expected [" + + key + + "] to be type [" + + boolean.class.getSimpleName() + + "] but found type [" + + value.getClass().getSimpleName() + + "]" + ); + }; + List filesData = new ArrayList<>(); + for (Object object : paths) { + Map file = new HashMap<>((Map) object); + String pathAsString = checkString.apply(file, "path"); + String relativePathAsString = checkString.apply(file, "relative_path"); + String relativeTo = checkString.apply(file, "relative_to"); + String pathSetting = checkString.apply(file, "path_setting"); + String settingBaseDirAsString = checkString.apply(file, "basedir_if_relative"); + String modeAsString = checkString.apply(file, "mode"); + String platformAsString = checkString.apply(file, "platform"); + Boolean exclusiveBoolean = checkBoolean.apply(file, "exclusive"); + boolean exclusive = exclusiveBoolean != null && exclusiveBoolean; + + if (file.isEmpty() == false) { + throw new PolicyValidationException("unknown key(s) [" + file + "] in a listed file for files entitlement"); + } + int foundKeys = (pathAsString != null ? 1 : 0) + (relativePathAsString != null ? 1 : 0) + (pathSetting != null ? 1 : 0); + if (foundKeys != 1) { + throw new PolicyValidationException( + "a files entitlement entry must contain one of " + "[path, relative_path, path_setting]" + ); + } + + if (modeAsString == null) { + throw new PolicyValidationException("files entitlement must contain 'mode' for every listed file"); + } + Mode mode = parseMode(modeAsString); + Platform platform = null; + if (platformAsString != null) { + platform = parsePlatform(platformAsString); + } + + if (relativeTo != null && relativePathAsString == null) { + throw new PolicyValidationException("'relative_to' may only be used with 'relative_path'"); + } + + if (settingBaseDirAsString != null && pathSetting == null) { + throw new PolicyValidationException("'basedir_if_relative' may only be used with 'path_setting'"); + } + + final FileData fileData; + if (relativePathAsString != null) { + if (relativeTo == null) { + throw new PolicyValidationException("files entitlement with a 'relative_path' must specify 'relative_to'"); + } + BaseDir baseDir = parseBaseDir(relativeTo); + Path relativePath = Path.of(relativePathAsString); + if (FileUtils.isAbsolutePath(relativePathAsString)) { + throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative"); + } + fileData = FileData.ofRelativePath(relativePath, baseDir, mode); + } else if (pathAsString != null) { + Path path = Path.of(pathAsString); + if (FileUtils.isAbsolutePath(pathAsString) == false) { + throw new PolicyValidationException("'path' [" + pathAsString + "] must be absolute"); + } + fileData = FileData.ofPath(path, mode); + } else if (pathSetting != null) { + if (settingBaseDirAsString == null) { + throw new PolicyValidationException("files entitlement with a 'path_setting' must specify 'basedir_if_relative'"); + } + BaseDir baseDir = parseBaseDir(settingBaseDirAsString); + fileData = FileData.ofPathSetting(pathSetting, baseDir, mode); + } else { + throw new AssertionError("File entry validation error"); + } + filesData.add(fileData.withPlatform(platform).withExclusive(exclusive)); + } + return new FilesEntitlement(filesData); + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/InboundNetworkEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/InboundNetworkEntitlement.java similarity index 77% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/InboundNetworkEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/InboundNetworkEntitlement.java index 482d4e5100c0b..04f1cda86b46a 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/InboundNetworkEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/InboundNetworkEntitlement.java @@ -7,12 +7,14 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; /** * Describes an entitlement for inbound network actions (listen/accept/receive) */ public record InboundNetworkEntitlement() implements Entitlement { - @ExternalEntitlement + @ExternalEntitlement(esModulesOnly = false) public InboundNetworkEntitlement {} } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/LoadNativeLibrariesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/LoadNativeLibrariesEntitlement.java similarity index 83% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/LoadNativeLibrariesEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/LoadNativeLibrariesEntitlement.java index 9a840c4e3e32e..b297685876925 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/LoadNativeLibrariesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/LoadNativeLibrariesEntitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; /** * An Entitlement to allow loading native libraries diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java new file mode 100644 index 0000000000000..c75ccf26d1432 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ManageThreadsEntitlement.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; + +public record ManageThreadsEntitlement() implements Entitlement { + @ExternalEntitlement(esModulesOnly = false) + public ManageThreadsEntitlement {} +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/OutboundNetworkEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/OutboundNetworkEntitlement.java similarity index 83% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/OutboundNetworkEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/OutboundNetworkEntitlement.java index 50d9a47f580e5..dbdd6840f2ebe 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/OutboundNetworkEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/OutboundNetworkEntitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; /** * Describes an entitlement for outbound network actions (connect/send) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java new file mode 100644 index 0000000000000..ccb84c4a68c97 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/ReadStoreAttributesEntitlement.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +/** + * Describes an entitlement for reading file store attributes (e.g. disk space) + */ +public record ReadStoreAttributesEntitlement() implements Entitlement {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/SetHttpsConnectionPropertiesEntitlement.java similarity index 84% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/SetHttpsConnectionPropertiesEntitlement.java index bb2f65def9e18..abfcfdf18db20 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/SetHttpsConnectionPropertiesEntitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; /** * An Entitlement to allow setting properties to a single Https connection after this has been created diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/WriteAllSystemPropertiesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/WriteAllSystemPropertiesEntitlement.java similarity index 83% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/WriteAllSystemPropertiesEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/WriteAllSystemPropertiesEntitlement.java index f0d1d14177332..f0b02e82d3cb5 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/WriteAllSystemPropertiesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/WriteAllSystemPropertiesEntitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; /** * An Entitlement to allow writing all properties such as system properties. diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/WriteSystemPropertiesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/WriteSystemPropertiesEntitlement.java similarity index 86% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/WriteSystemPropertiesEntitlement.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/WriteSystemPropertiesEntitlement.java index 654ebbda9dab3..b7818bb14030b 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/WriteSystemPropertiesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/WriteSystemPropertiesEntitlement.java @@ -7,7 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.entitlement.runtime.policy; +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement; import java.util.List; import java.util.Set; diff --git a/libs/entitlement/src/main19/java/org/elasticsearch/entitlement/runtime/api/Java19ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main19/java/org/elasticsearch/entitlement/runtime/api/Java19ElasticsearchEntitlementChecker.java new file mode 100644 index 0000000000000..307ceb213bbe4 --- /dev/null +++ b/libs/entitlement/src/main19/java/org/elasticsearch/entitlement/runtime/api/Java19ElasticsearchEntitlementChecker.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +import org.elasticsearch.entitlement.bridge.Java19EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; + +import java.lang.foreign.Addressable; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryAddress; +import java.lang.foreign.MemorySession; +import java.lang.invoke.MethodHandle; +import java.nio.file.Path; + +public class Java19ElasticsearchEntitlementChecker extends ElasticsearchEntitlementChecker implements Java19EntitlementChecker { + + public Java19ElasticsearchEntitlementChecker(PolicyManager policyManager) { + super(policyManager); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( + Class callerClass, + Linker that, + FunctionDescriptor function + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_Linker$downcallHandle( + Class callerClass, + Linker that, + Addressable address, + FunctionDescriptor function + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( + Class callerClass, + Linker that, + MethodHandle target, + FunctionDescriptor function, + MemorySession scope + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_MemorySegment$$ofAddress( + Class callerClass, + MemoryAddress address, + long byteSize, + MemorySession session + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, MemorySession session) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, MemorySession session) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkLoadingNativeLibraries(callerClass); + } +} diff --git a/libs/entitlement/src/main20/java/org/elasticsearch/entitlement/runtime/api/Java20ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main20/java/org/elasticsearch/entitlement/runtime/api/Java20ElasticsearchEntitlementChecker.java new file mode 100644 index 0000000000000..db4f3e5911414 --- /dev/null +++ b/libs/entitlement/src/main20/java/org/elasticsearch/entitlement/runtime/api/Java20ElasticsearchEntitlementChecker.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +import org.elasticsearch.entitlement.bridge.Java20EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentScope; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.spi.FileSystemProvider; + +public class Java20ElasticsearchEntitlementChecker extends ElasticsearchEntitlementChecker implements Java20EntitlementChecker { + + public Java20ElasticsearchEntitlementChecker(PolicyManager policyManager) { + super(policyManager); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( + Class callerClass, + Linker that, + FunctionDescriptor function, + Linker.Option... options + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_Linker$downcallHandle( + Class callerClass, + Linker that, + MemorySegment address, + FunctionDescriptor function, + Linker.Option... options + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( + Class callerClass, + Linker that, + MethodHandle target, + FunctionDescriptor function, + SegmentScope scope + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_MemorySegment$$ofAddress(Class callerClass, long address) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_MemorySegment$$ofAddress(Class callerClass, long address, long byteSize) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_MemorySegment$$ofAddress(Class callerClass, long address, long byteSize, SegmentScope scope) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_MemorySegment$$ofAddress( + Class callerClass, + long address, + long byteSize, + SegmentScope scope, + Runnable cleanupAction + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_layout_ValueLayouts$OfAddressImpl$asUnbounded(Class callerClass, ValueLayout.OfAddress that) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, SegmentScope scope) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, SegmentScope scope) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void checkReadAttributesIfExists( + Class callerClass, + FileSystemProvider that, + Path path, + Class type, + LinkOption... options + ) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkExists(Class callerClass, FileSystemProvider that, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } +} diff --git a/libs/entitlement/src/main21/java/org/elasticsearch/entitlement/runtime/api/Java21ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main21/java/org/elasticsearch/entitlement/runtime/api/Java21ElasticsearchEntitlementChecker.java index ecb830daf2f61..d2944d4f285c9 100644 --- a/libs/entitlement/src/main21/java/org/elasticsearch/entitlement/runtime/api/Java21ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main21/java/org/elasticsearch/entitlement/runtime/api/Java21ElasticsearchEntitlementChecker.java @@ -12,9 +12,116 @@ import org.elasticsearch.entitlement.bridge.Java21EntitlementChecker; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; +import java.lang.foreign.AddressLayout; +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.invoke.MethodHandle; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.spi.FileSystemProvider; +import java.util.function.Consumer; + public class Java21ElasticsearchEntitlementChecker extends ElasticsearchEntitlementChecker implements Java21EntitlementChecker { public Java21ElasticsearchEntitlementChecker(PolicyManager policyManager) { super(policyManager); } + + @Override + public void check$jdk_internal_foreign_layout_ValueLayouts$OfAddressImpl$withTargetLayout( + Class callerClass, + AddressLayout that, + MemoryLayout memoryLayout + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( + Class callerClass, + Linker that, + MemorySegment address, + FunctionDescriptor function, + Linker.Option... options + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( + Class callerClass, + Linker that, + FunctionDescriptor function, + Linker.Option... options + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( + Class callerClass, + Linker that, + MethodHandle target, + FunctionDescriptor function, + Arena arena, + Linker.Option... options + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret(Class callerClass, MemorySegment that, long newSize) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( + Class callerClass, + MemorySegment that, + long newSize, + Arena arena, + Consumer cleanup + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( + Class callerClass, + MemorySegment that, + Arena arena, + Consumer cleanup + ) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, Arena arena) { + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, Arena arena) { + policyManager.checkFileRead(callerClass, path); + policyManager.checkLoadingNativeLibraries(callerClass); + } + + @Override + public void checkReadAttributesIfExists( + Class callerClass, + FileSystemProvider that, + Path path, + Class type, + LinkOption... options + ) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void checkExists(Class callerClass, FileSystemProvider that, Path path, LinkOption... options) { + policyManager.checkFileRead(callerClass, path); + } } diff --git a/libs/entitlement/src/main22/java/org/elasticsearch/entitlement/runtime/api/Java22ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main22/java/org/elasticsearch/entitlement/runtime/api/Java22ElasticsearchEntitlementChecker.java index 2fe8988a54822..7ccd3781d5e21 100644 --- a/libs/entitlement/src/main22/java/org/elasticsearch/entitlement/runtime/api/Java22ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main22/java/org/elasticsearch/entitlement/runtime/api/Java22ElasticsearchEntitlementChecker.java @@ -12,98 +12,9 @@ import org.elasticsearch.entitlement.bridge.Java22EntitlementChecker; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; -import java.lang.foreign.AddressLayout; -import java.lang.foreign.Arena; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.Linker; -import java.lang.foreign.MemoryLayout; -import java.lang.foreign.MemorySegment; -import java.lang.invoke.MethodHandle; -import java.nio.file.Path; -import java.util.function.Consumer; - public class Java22ElasticsearchEntitlementChecker extends Java21ElasticsearchEntitlementChecker implements Java22EntitlementChecker { public Java22ElasticsearchEntitlementChecker(PolicyManager policyManager) { super(policyManager); } - - @Override - public void check$jdk_internal_foreign_layout_ValueLayouts$OfAddressImpl$withTargetLayout( - Class callerClass, - AddressLayout that, - MemoryLayout memoryLayout - ) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( - Class callerClass, - Linker that, - FunctionDescriptor function, - Linker.Option... options - ) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$jdk_internal_foreign_abi_AbstractLinker$downcallHandle( - Class callerClass, - Linker that, - MemorySegment address, - FunctionDescriptor function, - Linker.Option... options - ) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$jdk_internal_foreign_abi_AbstractLinker$upcallStub( - Class callerClass, - Linker that, - MethodHandle target, - FunctionDescriptor function, - Arena arena, - Linker.Option... options - ) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret(Class callerClass, MemorySegment that, long newSize) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( - Class callerClass, - MemorySegment that, - long newSize, - Arena arena, - Consumer cleanup - ) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$jdk_internal_foreign_AbstractMemorySegmentImpl$reinterpret( - Class callerClass, - MemorySegment that, - Arena arena, - Consumer cleanup - ) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, String name, Arena arena) { - policyManager.checkLoadingNativeLibraries(callerClass); - } - - @Override - public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class callerClass, Path path, Arena arena) { - // TODO: check filesystem entitlement READ - policyManager.checkLoadingNativeLibraries(callerClass); - } } diff --git a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java index 0e2cc0bf94f41..3818e1131223a 100644 --- a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java @@ -17,10 +17,4 @@ public class Java23ElasticsearchEntitlementChecker extends Java22ElasticsearchEn public Java23ElasticsearchEntitlementChecker(PolicyManager policyManager) { super(policyManager); } - - @Override - public void check$java_lang_Runtime$exit(Class callerClass, Runtime runtime, int status) { - // TODO: this is just an example, we shouldn't really override a method implemented in the superclass - super.check$java_lang_Runtime$exit(callerClass, runtime, status); - } } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/bridge/UtilTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/bridge/UtilTests.java new file mode 100644 index 0000000000000..e1bf161174bb3 --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/bridge/UtilTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.entitlement.bridge.UtilTests.MockSensitiveClass.mockSensitiveMethod; + +@ESTestCase.WithoutSecurityManager +public class UtilTests extends ESTestCase { + + public void testCallerClass() { + assertEquals(UtilTests.class, mockSensitiveMethod()); + } + + /** + * A separate class so the stack walk can discern the sensitive method's own class + * from that of its caller. + */ + static class MockSensitiveClass { + public static Class mockSensitiveMethod() { + return Util.getCallerClass(); + } + } + +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java index 1521c80341b9d..faa6424eabfc0 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java @@ -9,55 +9,91 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusiveFileEntitlement; +import org.elasticsearch.entitlement.runtime.policy.FileAccessTree.ExclusivePath; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import static org.elasticsearch.core.PathUtils.getDefaultFileSystem; +import static org.elasticsearch.entitlement.runtime.policy.FileAccessTree.buildExclusivePathList; +import static org.elasticsearch.entitlement.runtime.policy.FileAccessTree.normalizePath; +import static org.elasticsearch.entitlement.runtime.policy.Platform.WINDOWS; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +@ESTestCase.WithoutSecurityManager public class FileAccessTreeTests extends ESTestCase { static Path root; + static Settings settings; @BeforeClass public static void setupRoot() { root = createTempDir(); + settings = Settings.EMPTY; } private static Path path(String s) { return root.resolve(s); } + private static final PathLookup TEST_PATH_LOOKUP = new PathLookup( + Path.of("/home"), + Path.of("/config"), + new Path[] { Path.of("/data1"), Path.of("/data2") }, + new Path[] { Path.of("/shared1"), Path.of("/shared2") }, + Path.of("/tmp"), + pattern -> settings.getValues(pattern) + ); + public void testEmpty() { - var tree = new FileAccessTree(List.of()); + var tree = accessTree(FilesEntitlement.EMPTY, List.of()); assertThat(tree.canRead(path("path")), is(false)); assertThat(tree.canWrite(path("path")), is(false)); } public void testRead() { - var tree = new FileAccessTree(List.of(entitlement("foo", "read"))); + var tree = accessTree(entitlement("foo", "read"), List.of()); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canRead(path("foo/subdir")), is(true)); + assertThat(tree.canRead(path("food")), is(false)); assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canWrite(path("food")), is(false)); assertThat(tree.canRead(path("before")), is(false)); assertThat(tree.canRead(path("later")), is(false)); } public void testWrite() { - var tree = new FileAccessTree(List.of(entitlement("foo", "read_write"))); + var tree = accessTree(entitlement("foo", "read_write"), List.of()); assertThat(tree.canWrite(path("foo")), is(true)); assertThat(tree.canWrite(path("foo/subdir")), is(true)); + assertThat(tree.canWrite(path("food")), is(false)); assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canRead(path("food")), is(false)); assertThat(tree.canWrite(path("before")), is(false)); assertThat(tree.canWrite(path("later")), is(false)); } public void testTwoPaths() { - var tree = new FileAccessTree(List.of(entitlement("foo", "read"), entitlement("bar", "read"))); + var tree = accessTree(entitlement("foo", "read", "bar", "read"), List.of()); assertThat(tree.canRead(path("a")), is(false)); assertThat(tree.canRead(path("bar")), is(true)); assertThat(tree.canRead(path("bar/subdir")), is(true)); @@ -68,22 +104,383 @@ public void testTwoPaths() { } public void testReadWriteUnderRead() { - var tree = new FileAccessTree(List.of(entitlement("foo", "read"), entitlement("foo/bar", "read_write"))); + var tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"), List.of()); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(true)); + assertThat(tree.canWrite(path("foo/bar")), is(true)); + assertThat(tree.canRead(path("foo/baz")), is(true)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + } + + public void testPrunedPaths() { + var tree = accessTree(entitlement("foo", "read", "foo/baz", "read", "foo/bar", "read"), List.of()); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(true)); + assertThat(tree.canWrite(path("foo/bar")), is(false)); + assertThat(tree.canRead(path("foo/baz")), is(true)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + // also test a non-existent subpath + assertThat(tree.canRead(path("foo/barf")), is(true)); + assertThat(tree.canWrite(path("foo/barf")), is(false)); + + tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"), List.of()); assertThat(tree.canRead(path("foo")), is(true)); assertThat(tree.canWrite(path("foo")), is(false)); assertThat(tree.canRead(path("foo/bar")), is(true)); assertThat(tree.canWrite(path("foo/bar")), is(true)); + assertThat(tree.canRead(path("foo/baz")), is(true)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + } + + public void testPathAndFileWithSamePrefix() { + var tree = accessTree(entitlement("foo/bar/", "read", "foo/bar.xml", "read"), List.of()); + assertThat(tree.canRead(path("foo")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(true)); + assertThat(tree.canRead(path("foo/bar/baz")), is(true)); + assertThat(tree.canRead(path("foo/bar.xml")), is(true)); + assertThat(tree.canRead(path("foo/bar.txt")), is(false)); + } + + public void testReadWithRelativePath() { + for (var dir : List.of("home")) { + var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", dir)), List.of()); + assertThat(tree.canRead(path("foo")), is(false)); + + assertThat(tree.canRead(path("/" + dir + "/foo")), is(true)); + + assertThat(tree.canRead(path("/" + dir + "/foo/subdir")), is(true)); + assertThat(tree.canRead(path("/" + dir + "/food")), is(false)); + assertThat(tree.canWrite(path("/" + dir + "/foo")), is(false)); + + assertThat(tree.canRead(path("/" + dir)), is(false)); + assertThat(tree.canRead(path("/" + dir + "/before")), is(false)); + assertThat(tree.canRead(path("/" + dir + "/later")), is(false)); + } + } + + public void testWriteWithRelativePath() { + for (var dir : List.of("home")) { + var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", dir)), List.of()); + assertThat(tree.canWrite(path("/" + dir + "/foo")), is(true)); + assertThat(tree.canWrite(path("/" + dir + "/foo/subdir")), is(true)); + assertThat(tree.canWrite(path("/" + dir)), is(false)); + assertThat(tree.canWrite(path("/" + dir + "/food")), is(false)); + assertThat(tree.canRead(path("/" + dir + "/foo")), is(true)); + assertThat(tree.canRead(path("/" + dir)), is(false)); + + assertThat(tree.canWrite(path("/" + dir)), is(false)); + assertThat(tree.canWrite(path("/" + dir + "/before")), is(false)); + assertThat(tree.canWrite(path("/" + dir + "/later")), is(false)); + } + } + + public void testMultipleDataDirs() { + var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "data")), List.of()); + assertThat(tree.canWrite(path("/data1/foo")), is(true)); + assertThat(tree.canWrite(path("/data2/foo")), is(true)); + assertThat(tree.canWrite(path("/data3/foo")), is(false)); + assertThat(tree.canWrite(path("/data1/foo/subdir")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canWrite(path("/data1/food")), is(false)); + assertThat(tree.canRead(path("/data1/foo")), is(true)); + assertThat(tree.canRead(path("/data2/foo")), is(true)); + assertThat(tree.canRead(path("foo")), is(false)); + + assertThat(tree.canWrite(path("/data1")), is(false)); + assertThat(tree.canWrite(path("/data2")), is(false)); + assertThat(tree.canWrite(path("/config/before")), is(false)); + assertThat(tree.canWrite(path("/config/later")), is(false)); } public void testNormalizePath() { - var tree = new FileAccessTree(List.of(entitlement("foo/../bar", "read"))); + var tree = accessTree(entitlement("foo/../bar", "read"), List.of()); assertThat(tree.canRead(path("foo/../bar")), is(true)); + assertThat(tree.canRead(path("foo/../bar/")), is(true)); assertThat(tree.canRead(path("foo")), is(false)); assertThat(tree.canRead(path("")), is(false)); } - FileEntitlement entitlement(String path, String mode) { - Path p = path(path); - return new FileEntitlement(p.toString(), mode); + public void testNormalizeDirectorySeparatorWindows() { + assumeTrue("normalization of windows paths", WINDOWS.isCurrent()); + + assertThat(FileAccessTree.normalizePath(Path.of("C:\\a\\b")), equalTo("C:\\a\\b")); + assertThat(FileAccessTree.normalizePath(Path.of("C:/a.xml")), equalTo("C:\\a.xml")); + assertThat(FileAccessTree.normalizePath(Path.of("C:/a/b.txt")), equalTo("C:\\a\\b.txt")); + assertThat(FileAccessTree.normalizePath(Path.of("C:/a/c\\foo.txt")), equalTo("C:\\a\\c\\foo.txt")); + + var tree = accessTree( + entitlement("C:\\a\\b", "read", "C:/a.xml", "read", "C:/a/b.txt", "read", "C:/a/c\\foo.txt", "read"), + List.of() + ); + + assertThat(tree.canRead(Path.of("C:/a.xml")), is(true)); + assertThat(tree.canRead(Path.of("C:\\a.xml")), is(true)); + assertThat(tree.canRead(Path.of("C:/a/")), is(false)); + assertThat(tree.canRead(Path.of("C:/a/b.txt")), is(true)); + assertThat(tree.canRead(Path.of("C:/a/b/c.txt")), is(true)); + assertThat(tree.canRead(Path.of("C:\\a\\b\\c.txt")), is(true)); + assertThat(tree.canRead(Path.of("C:\\a\\c\\")), is(false)); + assertThat(tree.canRead(Path.of("C:\\a\\c\\foo.txt")), is(true)); + } + + public void testNormalizeTrailingSlashes() { + var tree = accessTree(entitlement("/trailing/slash/", "read", "/no/trailing/slash", "read"), List.of()); + assertThat(tree.canRead(path("/trailing/slash")), is(true)); + assertThat(tree.canRead(path("/trailing/slash/")), is(true)); + assertThat(tree.canRead(path("/trailing/slash.xml")), is(false)); + assertThat(tree.canRead(path("/trailing/slash/file.xml")), is(true)); + assertThat(tree.canRead(path("/no/trailing/slash")), is(true)); + assertThat(tree.canRead(path("/no/trailing/slash/")), is(true)); + assertThat(tree.canRead(path("/no/trailing/slash.xml")), is(false)); + assertThat(tree.canRead(path("/no/trailing/slash/file.xml")), is(true)); + } + + public void testForwardSlashes() { + String sep = getDefaultFileSystem().getSeparator(); + var tree = accessTree(entitlement("a/b", "read", "m" + sep + "n", "read"), List.of()); + + // Native separators work + assertThat(tree.canRead(path("a" + sep + "b")), is(true)); + assertThat(tree.canRead(path("m" + sep + "n")), is(true)); + + // Forward slashes also work + assertThat(tree.canRead(path("a/b")), is(true)); + assertThat(tree.canRead(path("m/n")), is(true)); + } + + public void testJdkAccess() { + Path jdkDir = Paths.get(System.getProperty("java.home")); + var confDir = jdkDir.resolve("conf"); + var tree = accessTree(FilesEntitlement.EMPTY, List.of()); + + assertThat(tree.canRead(confDir), is(true)); + assertThat(tree.canWrite(confDir), is(false)); + assertThat(tree.canRead(jdkDir), is(false)); + } + + @SuppressForbidden(reason = "don't care about the directory location in tests") + public void testFollowLinks() throws IOException { + assumeFalse("Windows requires admin right to create symbolic links", WINDOWS.isCurrent()); + + Path baseSourceDir = Files.createTempDirectory("fileaccess_source"); + Path source1Dir = baseSourceDir.resolve("source1"); + Files.createDirectory(source1Dir); + Path source2Dir = baseSourceDir.resolve("source2"); + Files.createDirectory(source2Dir); + + Path baseTargetDir = Files.createTempDirectory("fileaccess_target"); + Path readTarget = baseTargetDir.resolve("read_link"); + Path writeTarget = baseTargetDir.resolve("write_link"); + Files.createSymbolicLink(readTarget, source1Dir); + Files.createSymbolicLink(writeTarget, source2Dir); + var tree = accessTree(entitlement(readTarget.toString(), "read", writeTarget.toString(), "read_write"), List.of()); + + assertThat(tree.canRead(baseSourceDir), is(false)); + assertThat(tree.canRead(baseTargetDir), is(false)); + + assertThat(tree.canRead(readTarget), is(true)); + assertThat(tree.canWrite(readTarget), is(false)); + assertThat(tree.canRead(source1Dir), is(true)); + assertThat(tree.canWrite(source1Dir), is(false)); + + assertThat(tree.canRead(writeTarget), is(true)); + assertThat(tree.canWrite(writeTarget), is(true)); + assertThat(tree.canRead(source2Dir), is(true)); + assertThat(tree.canWrite(source2Dir), is(true)); + } + + public void testTempDirAccess() { + var tree = FileAccessTree.of("test-component", "test-module", FilesEntitlement.EMPTY, TEST_PATH_LOOKUP, null, List.of()); + assertThat(tree.canRead(TEST_PATH_LOOKUP.tempDir()), is(true)); + assertThat(tree.canWrite(TEST_PATH_LOOKUP.tempDir()), is(true)); + } + + public void testConfigDirAccess() { + var tree = FileAccessTree.of("test-component", "test-module", FilesEntitlement.EMPTY, TEST_PATH_LOOKUP, null, List.of()); + assertThat(tree.canRead(TEST_PATH_LOOKUP.configDir()), is(true)); + assertThat(tree.canWrite(TEST_PATH_LOOKUP.configDir()), is(false)); + } + + public void testBasicExclusiveAccess() { + var tree = accessTree(entitlement("foo", "read"), exclusivePaths("test-component", "test-module", "foo")); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + tree = accessTree(entitlement("foo", "read_write"), exclusivePaths("test-component", "test-module", "foo")); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(true)); + tree = accessTree(entitlement("foo", "read"), exclusivePaths("test-component", "diff-module", "foo/bar")); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canRead(path("foo/baz")), is(true)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(false)); + assertThat(tree.canWrite(path("foo/bar")), is(false)); + tree = accessTree( + entitlement("foo", "read", "foo.xml", "read", "foo/bar.xml", "read_write"), + exclusivePaths("test-component", "diff-module", "foo/bar", "foo/baz", "other") + ); + assertThat(tree.canRead(path("foo")), is(true)); + assertThat(tree.canWrite(path("foo")), is(false)); + assertThat(tree.canRead(path("foo.xml")), is(true)); + assertThat(tree.canWrite(path("foo.xml")), is(false)); + assertThat(tree.canRead(path("foo/baz")), is(false)); + assertThat(tree.canWrite(path("foo/baz")), is(false)); + assertThat(tree.canRead(path("foo/bar")), is(false)); + assertThat(tree.canWrite(path("foo/bar")), is(false)); + assertThat(tree.canRead(path("foo/bar.xml")), is(true)); + assertThat(tree.canWrite(path("foo/bar.xml")), is(true)); + assertThat(tree.canRead(path("foo/bar.baz")), is(true)); + assertThat(tree.canWrite(path("foo/bar.baz")), is(false)); + assertThat(tree.canRead(path("foo/biz/bar.xml")), is(true)); + assertThat(tree.canWrite(path("foo/biz/bar.xml")), is(false)); + } + + public void testInvalidExclusiveAccess() { + var tree = accessTree(entitlement("a", "read"), exclusivePaths("diff-component", "diff-module", "a/b")); + assertThat(tree.canRead(path("a")), is(true)); + assertThat(tree.canWrite(path("a")), is(false)); + assertThat(tree.canRead(path("a/b")), is(false)); + assertThat(tree.canWrite(path("a/b")), is(false)); + assertThat(tree.canRead(path("a/b/c")), is(false)); + assertThat(tree.canWrite(path("a/b/c")), is(false)); + tree = accessTree(entitlement("a/b", "read"), exclusivePaths("diff-component", "diff-module", "a")); + assertThat(tree.canRead(path("a")), is(false)); + assertThat(tree.canWrite(path("a")), is(false)); + assertThat(tree.canRead(path("a/b")), is(false)); + assertThat(tree.canWrite(path("a/b")), is(false)); + tree = accessTree(entitlement("a", "read"), exclusivePaths("diff-component", "diff-module", "a")); + assertThat(tree.canRead(path("a")), is(false)); + assertThat(tree.canWrite(path("a")), is(false)); + } + + public void testDuplicatePrunedPaths() { + List inputPaths = List.of("/a", "/a", "/a/b", "/a/b", "/b/c", "b/c/d", "b/c/d", "b/c/d", "e/f", "e/f"); + List outputPaths = List.of("/a", "/b/c", "b/c/d", "e/f"); + var actual = FileAccessTree.pruneSortedPaths(inputPaths.stream().map(p -> normalizePath(path(p))).toList()); + var expected = outputPaths.stream().map(p -> normalizePath(path(p))).toList(); + assertEquals(expected, actual); + } + + public void testDuplicateExclusivePaths() { + // Bunch o' handy definitions + var pathAB = path("/a/b"); + var pathCD = path("/c/d"); + var originalFileData = FileData.ofPath(pathAB, READ).withExclusive(true); + var fileDataWithWriteMode = FileData.ofPath(pathAB, READ_WRITE).withExclusive(true); + var original = new ExclusiveFileEntitlement("component1", "module1", new FilesEntitlement(List.of(originalFileData))); + var differentComponent = new ExclusiveFileEntitlement("component2", original.moduleName(), original.filesEntitlement()); + var differentModule = new ExclusiveFileEntitlement(original.componentName(), "module2", original.filesEntitlement()); + var differentPath = new ExclusiveFileEntitlement( + original.componentName(), + original.moduleName(), + new FilesEntitlement(List.of(FileData.ofPath(pathCD, originalFileData.mode()).withExclusive(originalFileData.exclusive()))) + ); + var differentMode = new ExclusiveFileEntitlement( + original.componentName(), + original.moduleName(), + new FilesEntitlement(List.of(fileDataWithWriteMode)) + ); + var differentPlatform = new ExclusiveFileEntitlement( + original.componentName(), + original.moduleName(), + new FilesEntitlement(List.of(originalFileData.withPlatform(WINDOWS))) + ); + var originalExclusivePath = new ExclusivePath("component1", Set.of("module1"), normalizePath(pathAB)); + + // Some basic tests + + assertEquals( + "Single element should trivially work", + List.of(originalExclusivePath), + buildExclusivePathList(List.of(original), TEST_PATH_LOOKUP) + ); + assertEquals( + "Two identical elements should be combined", + List.of(originalExclusivePath), + buildExclusivePathList(List.of(original, original), TEST_PATH_LOOKUP) + ); + + // Don't merge things we shouldn't + + var distinctEntitlements = List.of(original, differentComponent, differentModule, differentPath); + var distinctPaths = List.of( + originalExclusivePath, + new ExclusivePath("component2", Set.of(original.moduleName()), originalExclusivePath.path()), + new ExclusivePath(original.componentName(), Set.of("module2"), originalExclusivePath.path()), + new ExclusivePath(original.componentName(), Set.of(original.moduleName()), normalizePath(pathCD)) + ); + var iae = expectThrows(IllegalArgumentException.class, () -> buildExclusivePathList(distinctEntitlements, TEST_PATH_LOOKUP)); + var pathABString = pathAB.toAbsolutePath().toString(); + assertThat( + iae.getMessage(), + equalTo( + "Path [" + + pathABString + + "] is already exclusive to [component1][module1], cannot add exclusive access for [component2][module1]" + ) + ); + + var equivalentEntitlements = List.of(original, differentMode, differentPlatform); + var equivalentPaths = List.of(originalExclusivePath); + assertEquals( + "Exclusive paths should be combined even if the entitlements are different", + equivalentPaths, + buildExclusivePathList(equivalentEntitlements, TEST_PATH_LOOKUP) + ); + } + + public void testWindowsAbsolutPathAccess() { + assumeTrue("Specific to windows for paths with a root (DOS or UNC)", WINDOWS.isCurrent()); + + var fileAccessTree = FileAccessTree.of( + "test", + "test", + new FilesEntitlement( + List.of( + FileData.ofPath(Path.of("\\\\.\\pipe\\"), READ), + FileData.ofPath(Path.of("D:\\.gradle"), READ), + FileData.ofPath(Path.of("D:\\foo"), READ), + FileData.ofPath(Path.of("C:\\foo"), FilesEntitlement.Mode.READ_WRITE) + ) + ), + TEST_PATH_LOOKUP, + null, + List.of() + ); + + assertThat(fileAccessTree.canRead(Path.of("\\\\.\\pipe\\bar")), is(true)); + assertThat(fileAccessTree.canRead(Path.of("C:\\foo")), is(true)); + assertThat(fileAccessTree.canWrite(Path.of("C:\\foo")), is(true)); + assertThat(fileAccessTree.canRead(Path.of("D:\\foo")), is(true)); + assertThat(fileAccessTree.canWrite(Path.of("D:\\foo")), is(false)); + } + + FileAccessTree accessTree(FilesEntitlement entitlement, List exclusivePaths) { + return FileAccessTree.of("test-component", "test-module", entitlement, TEST_PATH_LOOKUP, null, exclusivePaths); + } + + static FilesEntitlement entitlement(String... values) { + List filesData = new ArrayList<>(); + for (int i = 0; i < values.length; i += 2) { + Map fileData = new HashMap<>(); + fileData.put("path", path(values[i]).toString()); + fileData.put("mode", values[i + 1]); + filesData.add(fileData); + } + return FilesEntitlement.build(filesData); + } + + static FilesEntitlement entitlement(Map value) { + return FilesEntitlement.build(List.of(value)); + } + + static List exclusivePaths(String componentName, String moduleName, String... paths) { + List exclusivePaths = new ArrayList<>(); + for (String path : paths) { + exclusivePaths.add(new ExclusivePath(componentName, Set.of(moduleName), normalizePath(path(path)))); + } + return exclusivePaths; } } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileUtilsTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileUtilsTests.java new file mode 100644 index 0000000000000..4c7e5e49d3ac8 --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileUtilsTests.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.entitlement.runtime.policy.FileUtils.PATH_ORDER; +import static org.elasticsearch.entitlement.runtime.policy.FileUtils.isAbsolutePath; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; + +public class FileUtilsTests extends ESTestCase { + + public void testPathIsAbsolute() { + var windowsNamedPipe = "\\\\.\\pipe"; + var windowsDosAbsolutePath = "C:\\temp"; + var unixAbsolutePath = "/tmp/foo"; + var unixStyleUncPath = "//C/temp"; + var uncPath = "\\\\C\\temp"; + var longPath = "\\\\?\\C:\\temp"; + + var relativePath = "foo"; + var headingSlashRelativePath = "\\foo"; + + assertThat(isAbsolutePath(windowsNamedPipe), is(true)); + assertThat(isAbsolutePath(windowsDosAbsolutePath), is(true)); + assertThat(isAbsolutePath(unixAbsolutePath), is(true)); + assertThat(isAbsolutePath(unixStyleUncPath), is(true)); + assertThat(isAbsolutePath(uncPath), is(true)); + assertThat(isAbsolutePath(longPath), is(true)); + + assertThat(isAbsolutePath(relativePath), is(false)); + assertThat(isAbsolutePath(headingSlashRelativePath), is(false)); + assertThat(isAbsolutePath(""), is(false)); + } + + public void testPathOrderPosix() { + assumeFalse("path ordering rules specific to non-Windows path styles", Platform.WINDOWS.isCurrent()); + + // Unix-style + // Directories come BEFORE files; note that this differs from natural lexicographical order + assertThat(PATH_ORDER.compare("/a/b", "/a.xml"), lessThan(0)); + + // Natural lexicographical order is respected in all the other cases + assertThat(PATH_ORDER.compare("/a/b", "/a/b.txt"), lessThan(0)); + assertThat(PATH_ORDER.compare("/a/c", "/a/b.txt"), greaterThan(0)); + assertThat(PATH_ORDER.compare("/a/b", "/a/b/foo.txt"), lessThan(0)); + + // Inverted-windows style + // Directories come BEFORE files; note that this differs from natural lexicographical order + assertThat(PATH_ORDER.compare("C:/a/b", "C:/a.xml"), lessThan(0)); + + // Natural lexicographical order is respected in all the other cases + assertThat(PATH_ORDER.compare("C:/a/b", "C:/a/b.txt"), lessThan(0)); + assertThat(PATH_ORDER.compare("C:/a/c", "C:/a/b.txt"), greaterThan(0)); + assertThat(PATH_ORDER.compare("C:/a/b", "C:/a/b/foo.txt"), lessThan(0)); + + // "\" is a valid file name character on Posix, test we treat it like that + assertThat(PATH_ORDER.compare("/a\\b", "/a/b.txt"), greaterThan(0)); + } + + public void testPathOrderWindows() { + assumeTrue("path ordering rules specific to Windows", Platform.WINDOWS.isCurrent()); + + // Directories come BEFORE files; note that this differs from natural lexicographical order + assertThat(PATH_ORDER.compare("C:\\a\\b", "C:\\a.xml"), lessThan(0)); + + // Natural lexicographical order is respected in all the other cases + assertThat(PATH_ORDER.compare("C:\\a\\b", "C:\\a\\b.txt"), lessThan(0)); + assertThat(PATH_ORDER.compare("C:\\a\\b", "C:\\a\\b\\foo.txt"), lessThan(0)); + assertThat(PATH_ORDER.compare("C:\\a\\c", "C:\\a\\b.txt"), greaterThan(0)); + } + + public void testPathOrderingSpecialCharacters() { + assertThat(PATH_ORDER.compare("aa\uD801\uDC28", "aa\uD801\uDC28"), is(0)); + assertThat(PATH_ORDER.compare("aa\uD801\uDC28", "aa\uD801\uDC28a"), lessThan(0)); + + var s = PathUtils.getDefaultFileSystem().getSeparator(); + // Similarly to the other tests, we assert that Directories come BEFORE files, even when names are special characters + assertThat(PATH_ORDER.compare(s + "\uD801\uDC28" + s + "b", s + "\uD801\uDC28.xml"), lessThan(0)); + assertThat(PATH_ORDER.compare(s + "\uD801\uDC28" + s + "b", s + "b.xml"), greaterThan(0)); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java index 20035d0bb258b..4e8ac7c547984 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -9,15 +9,21 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; import org.elasticsearch.entitlement.runtime.policy.PolicyManager.ModuleEntitlements; import org.elasticsearch.entitlement.runtime.policy.agent.TestAgent; import org.elasticsearch.entitlement.runtime.policy.agent.inner.TestInnerAgent; +import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; import org.elasticsearch.test.jar.JarUtils; import org.junit.BeforeClass; import java.io.IOException; +import java.lang.StackWalker.StackFrame; import java.lang.module.Configuration; import java.lang.module.ModuleFinder; import java.net.URL; @@ -31,7 +37,10 @@ import static java.util.Map.entry; import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.SERVER_COMPONENT_NAME; import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; @@ -49,77 +58,128 @@ public class PolicyManagerTests extends ESTestCase { */ private static Module NO_ENTITLEMENTS_MODULE; + private static Path TEST_BASE_DIR; + + private static PathLookup TEST_PATH_LOOKUP; + @BeforeClass public static void beforeClass() { try { // Any old module will do for tests using NO_ENTITLEMENTS_MODULE NO_ENTITLEMENTS_MODULE = makeClassInItsOwnModule().getModule(); + + TEST_BASE_DIR = createTempDir().toAbsolutePath(); + TEST_PATH_LOOKUP = new PathLookup( + TEST_BASE_DIR.resolve("/user/home"), + TEST_BASE_DIR.resolve("/config"), + new Path[] { TEST_BASE_DIR.resolve("/data1/"), TEST_BASE_DIR.resolve("/data2") }, + new Path[] { TEST_BASE_DIR.resolve("/shared1"), TEST_BASE_DIR.resolve("/shared2") }, + TEST_BASE_DIR.resolve("/temp"), + Settings.EMPTY::getValues + ); } catch (Exception e) { throw new IllegalStateException(e); } - } public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() { + var plugin1SourcePath = Path.of("modules", "plugin1"); var policyManager = new PolicyManager( createEmptyTestServerPolicy(), List.of(), Map.of("plugin1", createPluginPolicy("plugin.module")), c -> "plugin1", + Map.of("plugin1", plugin1SourcePath), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertEquals("No policy for the unnamed module", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + assertEquals( + "No policy for the unnamed module", + policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName()), + policyManager.getEntitlements(callerClass) + ); - assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + assertEquals( + Map.of(requestingModule, policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName())), + policyManager.moduleEntitlementsMap + ); } public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() { + var plugin1SourcePath = Path.of("modules", "plugin1"); var policyManager = new PolicyManager( createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", + Map.of("plugin1", plugin1SourcePath), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertEquals("No policy for this plugin", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + assertEquals( + "No policy for this plugin", + policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName()), + policyManager.getEntitlements(callerClass) + ); - assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + assertEquals( + Map.of(requestingModule, policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName())), + policyManager.moduleEntitlementsMap + ); } public void testGetEntitlementsFailureIsCached() { + var plugin1SourcePath = Path.of("modules", "plugin1"); var policyManager = new PolicyManager( createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", + Map.of("plugin1", plugin1SourcePath), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); - assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + assertEquals( + policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName()), + policyManager.getEntitlements(callerClass) + ); + assertEquals( + Map.of(requestingModule, policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName())), + policyManager.moduleEntitlementsMap + ); // A second time - assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + assertEquals( + policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName()), + policyManager.getEntitlements(callerClass) + ); // Nothing new in the map - assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + assertEquals( + Map.of(requestingModule, policyManager.defaultEntitlements("plugin1", plugin1SourcePath, requestingModule.getName())), + policyManager.moduleEntitlementsMap + ); } public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { @@ -128,8 +188,11 @@ public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { List.of(), Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), c -> "plugin2", + Map.of("plugin2", Path.of("modules", "plugin2")), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -145,8 +208,11 @@ public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotF List.of(), Map.of(), c -> null, + Map.of(), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Tests do not run modular, so we cannot use a server class. @@ -154,11 +220,22 @@ public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotF // So we use a random module in the boot layer, and a random class from that module (not java.base -- it is // loaded too early) to mimic a class that would be in the server module. var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); + var mockServerSourcePath = PolicyManager.getComponentPathFromClass(mockServerClass); var requestingModule = mockServerClass.getModule(); - assertEquals("No policy for this module in server", ModuleEntitlements.NONE, policyManager.getEntitlements(mockServerClass)); + assertEquals( + "No policy for this module in server", + policyManager.defaultEntitlements(SERVER_COMPONENT_NAME, mockServerSourcePath, requestingModule.getName()), + policyManager.getEntitlements(mockServerClass) + ); - assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + assertEquals( + Map.of( + requestingModule, + policyManager.defaultEntitlements(SERVER_COMPONENT_NAME, mockServerSourcePath, requestingModule.getName()) + ), + policyManager.moduleEntitlementsMap + ); } public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException { @@ -167,8 +244,11 @@ public void testGetEntitlementsReturnsEntitlementsForServerModule() throws Class List.of(), Map.of(), c -> null, + Map.of(), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Tests do not run modular, so we cannot use a server class. @@ -192,8 +272,11 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc List.of(), Map.of("mock-plugin", createPluginPolicy("org.example.plugin")), c -> "mock-plugin", + Map.of("mock-plugin", Path.of("modules", "mock-plugin")), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); var layer = createLayerForJar(jar, "org.example.plugin"); @@ -201,8 +284,7 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc var entitlements = policyManager.getEntitlements(mockPluginClass); assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); - // TODO: this can't work on Windows, we need to have the root be unknown - // assertThat(entitlements.fileAccess().canRead("/test/path"), is(true)); + assertThat(entitlements.fileAccess().canRead(TEST_BASE_DIR), is(true)); } public void testGetEntitlementsResultIsCached() { @@ -211,8 +293,11 @@ public void testGetEntitlementsResultIsCached() { List.of(), Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), c -> "plugin2", + Map.of("plugin2", Path.of("modules", "plugin2")), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -235,7 +320,6 @@ public void testRequestingClassFastPath() throws IOException, ClassNotFoundExcep } public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException { - var agentsClass = new TestAgent(); var entitlementsClass = makeClassInItsOwnModule(); // A class in the entitlements library itself var requestingClass = makeClassInItsOwnModule(); // This guy is always the right answer var instrumentedClass = makeClassInItsOwnModule(); // The class that called the check method @@ -246,18 +330,21 @@ public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoun assertEquals( "Skip entitlement library and the instrumented method", requestingClass, - policyManager.findRequestingClass(Stream.of(entitlementsClass, instrumentedClass, requestingClass, ignorableClass)).orElse(null) + policyManager.findRequestingFrame( + Stream.of(entitlementsClass, instrumentedClass, requestingClass, ignorableClass).map(MockFrame::new) + ).map(StackFrame::getDeclaringClass).orElse(null) ); assertEquals( "Skip multiple library frames", requestingClass, - policyManager.findRequestingClass(Stream.of(entitlementsClass, entitlementsClass, instrumentedClass, requestingClass)) - .orElse(null) + policyManager.findRequestingFrame( + Stream.of(entitlementsClass, entitlementsClass, instrumentedClass, requestingClass).map(MockFrame::new) + ).map(StackFrame::getDeclaringClass).orElse(null) ); assertThrows( "Non-modular caller frames are not supported", NullPointerException.class, - () -> policyManager.findRequestingClass(Stream.of(entitlementsClass, null)) + () -> policyManager.findRequestingFrame(Stream.of(entitlementsClass, null).map(MockFrame::new)) ); } @@ -269,9 +356,12 @@ public void testAgentsEntitlements() throws IOException, ClassNotFoundException createEmptyTestServerPolicy(), List.of(new CreateClassLoaderEntitlement()), Map.of(), - c -> "test", + c -> c.getPackageName().startsWith(TEST_AGENTS_PACKAGE_NAME) ? null : "test", + Map.of(), TEST_AGENTS_PACKAGE_NAME, - NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() ); ModuleEntitlements agentsEntitlements = policyManager.getEntitlements(TestAgent.class); assertThat(agentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); @@ -286,14 +376,220 @@ public void testAgentsEntitlements() throws IOException, ClassNotFoundException } } - private static Class makeClassInItsOwnModule() throws IOException, ClassNotFoundException { - final Path home = createTempDir(); - Path jar = createMockPluginJar(home); - var layer = createLayerForJar(jar, "org.example.plugin"); - return layer.findLoader("org.example.plugin").loadClass("q.B"); + public void testDuplicateEntitlements() { + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + new Policy( + "server", + List.of(new Scope("test", List.of(new CreateClassLoaderEntitlement(), new CreateClassLoaderEntitlement()))) + ), + List.of(), + Map.of(), + c -> "test", + Map.of(), + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertEquals( + "[(server)] using module [test] found duplicate entitlement " + "[" + CreateClassLoaderEntitlement.class.getName() + "]", + iae.getMessage() + ); + + iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + createEmptyTestServerPolicy(), + List.of(new CreateClassLoaderEntitlement(), new CreateClassLoaderEntitlement()), + Map.of(), + c -> "test", + Map.of(), + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertEquals( + "[(APM agent)] using module [ALL-UNNAMED] found duplicate entitlement " + + "[" + + CreateClassLoaderEntitlement.class.getName() + + "]", + iae.getMessage() + ); + + iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of( + "plugin1", + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + FilesEntitlement.EMPTY, + new CreateClassLoaderEntitlement(), + new FilesEntitlement( + List.of(FilesEntitlement.FileData.ofPath(Path.of("/tmp/test"), FilesEntitlement.Mode.READ)) + ) + ) + ) + ) + ) + ), + c -> "plugin1", + Map.of("plugin1", Path.of("modules", "plugin1")), + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertEquals( + "[plugin1] using module [test] found duplicate entitlement " + "[" + FilesEntitlement.class.getName() + "]", + iae.getMessage() + ); } - private static Class makeClassInItsOwnUnnamedModule() throws IOException, ClassNotFoundException { + public void testFilesEntitlementsWithExclusive() { + var baseTestPath = Path.of("/base").toAbsolutePath(); + var testPath1 = Path.of("/base/test").toAbsolutePath(); + var testPath2 = Path.of("/base/test/foo").toAbsolutePath(); + var iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of( + "plugin1", + new Policy( + "test", + List.of( + new Scope( + "test.module1", + List.of( + new FilesEntitlement( + List.of(FilesEntitlement.FileData.ofPath(testPath1, FilesEntitlement.Mode.READ).withExclusive(true)) + ) + ) + ) + ) + ), + "plugin2", + new Policy( + "test", + List.of( + new Scope( + "test.module2", + List.of( + new FilesEntitlement( + List.of(FilesEntitlement.FileData.ofPath(testPath1, FilesEntitlement.Mode.READ).withExclusive(true)) + ) + ) + ) + ) + ) + ), + c -> "", + Map.of("plugin1", Path.of("modules", "plugin1"), "plugin2", Path.of("modules", "plugin2")), + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertThat( + iae.getMessage(), + allOf( + containsString("Path [" + testPath1 + "] is already exclusive"), + containsString("[plugin1][test.module1]"), + containsString("[plugin2][test.module2]"), + containsString("cannot add exclusive access") + ) + ); + + iae = expectThrows( + IllegalArgumentException.class, + () -> new PolicyManager( + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(testPath2, FilesEntitlement.Mode.READ).withExclusive(true), + FilesEntitlement.FileData.ofPath(baseTestPath, FilesEntitlement.Mode.READ) + ) + ) + ) + ) + ) + ), + List.of(), + Map.of( + "plugin1", + new Policy( + "test", + List.of( + new Scope( + "test", + List.of( + new FilesEntitlement( + List.of(FilesEntitlement.FileData.ofPath(testPath1, FilesEntitlement.Mode.READ).withExclusive(true)) + ) + ) + ) + ) + ) + ), + c -> "", + Map.of(), + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ) + ); + assertEquals( + Strings.format( + "duplicate/overlapping exclusive paths found in files entitlements: " + + "[[plugin1] [test] [%s]] and [[(server)] [test] [%s]]", + testPath1, + testPath2 + ), + iae.getMessage() + ); + } + + /** + * If the plugin resolver tells us a class is in a plugin, don't conclude that it's in an agent. + */ + public void testPluginResolverOverridesAgents() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + List.of(new CreateClassLoaderEntitlement()), + Map.of(), + c -> "test", // Insist that the class is in a plugin + Map.of(), + TEST_AGENTS_PACKAGE_NAME, + NO_ENTITLEMENTS_MODULE, + TEST_PATH_LOOKUP, + Set.of() + ); + ModuleEntitlements notAgentsEntitlements = policyManager.getEntitlements(TestAgent.class); + assertThat(notAgentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(false)); + } + + private static Class makeClassInItsOwnModule() throws IOException, ClassNotFoundException { final Path home = createTempDir(); Path jar = createMockPluginJar(home); var layer = createLayerForJar(jar, "org.example.plugin"); @@ -301,7 +597,17 @@ private static Class makeClassInItsOwnUnnamedModule() throws IOException, Cla } private static PolicyManager policyManager(String agentsPackageName, Module entitlementsModule) { - return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", agentsPackageName, entitlementsModule); + return new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of(), + c -> "test", + Map.of(), + agentsPackageName, + entitlementsModule, + TEST_PATH_LOOKUP, + Set.of() + ); } private static Policy createEmptyTestServerPolicy() { @@ -319,7 +625,10 @@ private static Policy createPluginPolicy(String... pluginModules) { .map( name -> new Scope( name, - List.of(new FileEntitlement("/test/path", FileEntitlement.Mode.READ), new CreateClassLoaderEntitlement()) + List.of( + new FilesEntitlement(List.of(FilesEntitlement.FileData.ofPath(TEST_BASE_DIR, FilesEntitlement.Mode.READ))), + new CreateClassLoaderEntitlement() + ) ) ) .toList() @@ -361,4 +670,47 @@ private static ModuleLayer createLayerForJar(Path jar, String moduleName) { ); return moduleController.layer(); } + + record MockFrame(Class declaringClass) implements StackFrame { + @Override + public String getClassName() { + return getDeclaringClass().getName(); + } + + @Override + public String getMethodName() { + throw new UnsupportedOperationException(); + } + + @Override + public Class getDeclaringClass() { + return declaringClass; + } + + @Override + public int getByteCodeIndex() { + throw new UnsupportedOperationException(); + } + + @Override + public String getFileName() { + throw new UnsupportedOperationException(); + } + + @Override + public int getLineNumber() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isNativeMethod() { + throw new UnsupportedOperationException(); + } + + @Override + public StackTraceElement toStackTraceElement() { + throw new UnsupportedOperationException(); + } + } + } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java index cc8043990930d..f0ff9211d35b4 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java @@ -40,22 +40,56 @@ public void testEntitlementDoesNotExist() { public void testEntitlementMissingParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - - file: {} + - files: + - path: test-path """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( - "[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " - + "for entitlement type [file]: missing entitlement parameter [path]", + "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [files]: files entitlement must contain 'mode' for every listed file", ppe.getMessage() ); + } + + public void testEntitlementMissingDependentParameter() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - files: + - relative_path: test-path + mode: read + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); + assertEquals( + "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [files]: files entitlement with a 'relative_path' must specify 'relative_to'", + ppe.getMessage() + ); + } - ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + public void testEntitlementMutuallyExclusiveParameters() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - files: + - relative_path: test-path + path: test-path + mode: read + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); + assertEquals( + "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [files]: a files entitlement entry must contain one of " + + "[path, relative_path, path_setting]", + ppe.getMessage() + ); + } + + public void testEntitlementAtLeastOneParameter() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - - file: - path: test-path + - files: + - mode: read """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( - "[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " - + "for entitlement type [file]: missing entitlement parameter [mode]", + "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [files]: a files entitlement entry must contain one of " + + "[path, relative_path, path_setting]", ppe.getMessage() ); } @@ -63,14 +97,14 @@ public void testEntitlementMissingParameter() { public void testEntitlementExtraneousParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - - file: - path: test-path - mode: read - extra: test + - files: + - path: test-path + mode: read + extra: test """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( - "[6:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " - + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}", + "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [files]: unknown key(s) [{extra=test}] in a listed file for files entitlement", ppe.getMessage() ); } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index 191b3afcdc674..8518b60f0ed01 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -9,19 +9,40 @@ package org.elasticsearch.entitlement.runtime.policy; +import org.elasticsearch.core.Strings; +import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Set; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +@ESTestCase.WithoutSecurityManager public class PolicyParserTests extends ESTestCase { + public static String TEST_ABSOLUTE_PATH_TO_FILE; + + @BeforeClass + public static void beforeClass() throws IOException { + TEST_ABSOLUTE_PATH_TO_FILE = createTempFile().toAbsolutePath().toString(); + } + private static class TestWrongEntitlementName implements Entitlement {} public static class ManyConstructorsEntitlement implements Entitlement { @@ -32,36 +53,167 @@ public ManyConstructorsEntitlement(String s) {} public ManyConstructorsEntitlement(int i) {} } - public void testGetEntitlementTypeName() { - assertEquals("create_class_loader", PolicyParser.getEntitlementTypeName(CreateClassLoaderEntitlement.class)); + public static class ManyMethodsEntitlement implements Entitlement { + @ExternalEntitlement + public static ManyMethodsEntitlement create(String s) { + return new ManyMethodsEntitlement(); + } + + @ExternalEntitlement + public static ManyMethodsEntitlement create(int i) { + return new ManyMethodsEntitlement(); + } + } + + public static class ConstructorAndMethodEntitlement implements Entitlement { + @ExternalEntitlement + public static ConstructorAndMethodEntitlement create(String s) { + return new ConstructorAndMethodEntitlement(s); + } + + @ExternalEntitlement + public ConstructorAndMethodEntitlement(String s) {} + } + + public static class NonStaticMethodEntitlement implements Entitlement { + @ExternalEntitlement + public NonStaticMethodEntitlement create() { + return new NonStaticMethodEntitlement(); + } + } - var ex = expectThrows(IllegalArgumentException.class, () -> PolicyParser.getEntitlementTypeName(TestWrongEntitlementName.class)); + public void testBuildEntitlementNameFromClass() { + assertEquals("create_class_loader", PolicyParser.buildEntitlementNameFromClass(CreateClassLoaderEntitlement.class)); + + var ex = expectThrows( + IllegalArgumentException.class, + () -> PolicyParser.buildEntitlementNameFromClass(TestWrongEntitlementName.class) + ); assertThat( ex.getMessage(), equalTo("TestWrongEntitlementName is not a valid Entitlement class name. A valid class name must end with 'Entitlement'") ); } + private static InputStream createFilesTestPolicy() { + return new ByteArrayInputStream(Strings.format(""" + entitlement-module-name: + - files: + - path: '%s' + mode: "read_write" + """, TEST_ABSOLUTE_PATH_TO_FILE).getBytes(StandardCharsets.UTF_8)); + } + public void testPolicyBuilder() throws IOException { - Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false) - .parsePolicy(); + Policy parsedPolicy = new PolicyParser(createFilesTestPolicy(), "test-policy.yaml", false).parsePolicy(); Policy expected = new Policy( "test-policy.yaml", - List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", "read_write")))) + List.of( + new Scope( + "entitlement-module-name", + List.of(FilesEntitlement.build(List.of(Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write")))) + ) + ) ); assertEquals(expected, parsedPolicy); } public void testPolicyBuilderOnExternalPlugin() throws IOException { - Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true) - .parsePolicy(); + Policy parsedPolicy = new PolicyParser(createFilesTestPolicy(), "test-policy.yaml", true).parsePolicy(); Policy expected = new Policy( "test-policy.yaml", - List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", "read_write")))) + List.of( + new Scope( + "entitlement-module-name", + List.of(FilesEntitlement.build(List.of(Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write")))) + ) + ) ); assertEquals(expected, parsedPolicy); } + public void testParseFiles() throws IOException { + Policy policyWithOnePath = new PolicyParser(createFilesTestPolicy(), "test-policy.yaml", false).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of( + new Scope( + "entitlement-module-name", + List.of(FilesEntitlement.build(List.of(Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write")))) + ) + ) + ); + assertEquals(expected, policyWithOnePath); + + String testPathToReadDir = createTempDir().toAbsolutePath().toString(); + Policy policyWithTwoPaths = new PolicyParser(new ByteArrayInputStream(Strings.format(""" + entitlement-module-name: + - files: + - path: '%s' + mode: "read_write" + - path: '%s' + mode: "read" + """, TEST_ABSOLUTE_PATH_TO_FILE, testPathToReadDir).getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); + expected = new Policy( + "test-policy.yaml", + List.of( + new Scope( + "entitlement-module-name", + List.of( + FilesEntitlement.build( + List.of( + Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"), + Map.of("path", testPathToReadDir, "mode", "read") + ) + ) + ) + ) + ) + ); + assertEquals(expected, policyWithTwoPaths); + + String relativePathToFile = Path.of("test/path/to/file").normalize().toString(); + String relativePathToDir = Path.of("test/path/to/read-dir/").normalize().toString(); + Policy policyWithMultiplePathsAndBaseDir = new PolicyParser( + new ByteArrayInputStream(Strings.format(""" + entitlement-module-name: + - files: + - relative_path: '%s' + relative_to: "data" + mode: "read_write" + - relative_path: '%s' + relative_to: "config" + mode: "read" + - path: '%s' + mode: "read_write" + - path_setting: foo.bar + basedir_if_relative: config + mode: read + """, relativePathToFile, relativePathToDir, TEST_ABSOLUTE_PATH_TO_FILE).getBytes(StandardCharsets.UTF_8)), + "test-policy.yaml", + false + ).parsePolicy(); + expected = new Policy( + "test-policy.yaml", + List.of( + new Scope( + "entitlement-module-name", + List.of( + FilesEntitlement.build( + List.of( + Map.of("relative_path", relativePathToFile, "mode", "read_write", "relative_to", "data"), + Map.of("relative_path", relativePathToDir, "mode", "read", "relative_to", "config"), + Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"), + Map.of("path_setting", "foo.bar", "basedir_if_relative", "config", "mode", "read") + ) + ) + ) + ) + ) + ); + assertEquals(expected, policyWithMultiplePathsAndBaseDir); + } + public void testParseNetwork() throws IOException { Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: @@ -145,6 +297,98 @@ public void testParseLoadNativeLibraries() throws IOException { assertEquals(expected, parsedPolicy); } + public void testVersionedPolicyParsing() throws IOException { + var versionedPolicy = new ByteArrayInputStream(""" + versions: + - x + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """.getBytes(StandardCharsets.UTF_8)); + + var policyParser = new PolicyParser(versionedPolicy, "test-policy.yaml", true); + var parsedPolicy = policyParser.parseVersionedPolicy(); + + Policy expectedPolicy = new Policy( + "test-policy.yaml", + List.of( + new Scope("entitlement-module-name", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("entitlement-module-name-2", List.of(new SetHttpsConnectionPropertiesEntitlement())) + ) + ); + assertEquals(expectedPolicy, parsedPolicy.policy()); + assertThat(parsedPolicy.versions(), contains("x")); + } + + public void testVersionedPolicyParsingMultipleVersions() throws IOException { + var versionedPolicy = new ByteArrayInputStream(""" + versions: + - x + - y + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """.getBytes(StandardCharsets.UTF_8)); + + var policyParser = new PolicyParser(versionedPolicy, "test-policy.yaml", true); + var parsedPolicy = policyParser.parseVersionedPolicy(); + + Policy expectedPolicy = new Policy( + "test-policy.yaml", + List.of( + new Scope("entitlement-module-name", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("entitlement-module-name-2", List.of(new SetHttpsConnectionPropertiesEntitlement())) + ) + ); + assertEquals(expectedPolicy, parsedPolicy.policy()); + assertThat(parsedPolicy.versions(), contains("x", "y")); + } + + public void testVersionedPolicyParsingAnyFieldOrder() throws IOException { + var versionedPolicy = new ByteArrayInputStream(""" + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + versions: + - x + - y + """.getBytes(StandardCharsets.UTF_8)); + + var policyParser = new PolicyParser(versionedPolicy, "test-policy.yaml", true); + var parsedPolicy = policyParser.parseVersionedPolicy(); + + Policy expectedPolicy = new Policy( + "test-policy.yaml", + List.of( + new Scope("entitlement-module-name", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("entitlement-module-name-2", List.of(new SetHttpsConnectionPropertiesEntitlement())) + ) + ); + assertEquals(expectedPolicy, parsedPolicy.policy()); + assertThat(parsedPolicy.versions(), contains("x", "y")); + } + + public void testVersionedPolicyParsingEmptyPolicy() throws IOException { + var versionedPolicy = new ByteArrayInputStream(""" + versions: + - x + - y + """.getBytes(StandardCharsets.UTF_8)); + + var policyParser = new PolicyParser(versionedPolicy, "test-policy.yaml", true); + var parsedPolicy = policyParser.parseVersionedPolicy(); + + Policy expectedPolicy = new Policy("test-policy.yaml", List.of()); + assertEquals(expectedPolicy, parsedPolicy.policy()); + assertThat(parsedPolicy.versions(), contains("x", "y")); + } + public void testMultipleConstructorsAnnotated() throws IOException { var parser = new PolicyParser( new ByteArrayInputStream(""" @@ -166,4 +410,60 @@ public void testMultipleConstructorsAnnotated() throws IOException { ) ); } + + public void testMultipleMethodsAnnotated() throws IOException { + var parser = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - many_methods + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true, Map.of("many_methods", ManyMethodsEntitlement.class)); + + var e = expectThrows(IllegalStateException.class, parser::parsePolicy); + assertThat( + e.getMessage(), + equalTo( + "entitlement class " + + "[org.elasticsearch.entitlement.runtime.policy.PolicyParserTests$ManyMethodsEntitlement]" + + " has more than one constructor and/or method annotated with ExternalEntitlement" + ) + ); + } + + public void testConstructorAndMethodAnnotated() throws IOException { + var parser = new PolicyParser( + new ByteArrayInputStream(""" + entitlement-module-name: + - constructor_and_method + """.getBytes(StandardCharsets.UTF_8)), + "test-policy.yaml", + true, + Map.of("constructor_and_method", ConstructorAndMethodEntitlement.class) + ); + + var e = expectThrows(IllegalStateException.class, parser::parsePolicy); + assertThat( + e.getMessage(), + equalTo( + "entitlement class " + + "[org.elasticsearch.entitlement.runtime.policy.PolicyParserTests$ConstructorAndMethodEntitlement]" + + " has more than one constructor and/or method annotated with ExternalEntitlement" + ) + ); + } + + public void testNonStaticMethodAnnotated() throws IOException { + var parser = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - non_static + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true, Map.of("non_static", NonStaticMethodEntitlement.class)); + + var e = expectThrows(IllegalStateException.class, parser::parsePolicy); + assertThat( + e.getMessage(), + equalTo( + "entitlement class " + + "[org.elasticsearch.entitlement.runtime.policy.PolicyParserTests$NonStaticMethodEntitlement]" + + " has non-static method annotated with ExternalEntitlement" + ) + ); + } } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtilsTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtilsTests.java new file mode 100644 index 0000000000000..8ee0ce3736a8d --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtilsTests.java @@ -0,0 +1,386 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteAllSystemPropertiesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement; +import org.elasticsearch.test.ESTestCase; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.SEPARATOR; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +@ESTestCase.WithoutSecurityManager +public class PolicyUtilsTests extends ESTestCase { + + public void testCreatePluginPolicyWithPatch() { + + var policyPatch = """ + versions: + - 9.0.0 + - 9.0.0-SNAPSHOT + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """; + var base64EncodedPolicy = new String( + Base64.getEncoder().encode(policyPatch.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8 + ); + final Policy expectedPolicy = new Policy( + "test-plugin", + List.of( + new Scope("entitlement-module-name", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("entitlement-module-name-2", List.of(new SetHttpsConnectionPropertiesEntitlement())) + ) + ); + + var policy = PolicyUtils.parseEncodedPolicyIfExists( + base64EncodedPolicy, + "9.0.0", + true, + "test-plugin", + Set.of("entitlement-module-name", "entitlement-module-name-2") + ); + + assertThat(policy, equalTo(expectedPolicy)); + } + + public void testCreatePluginPolicyWithPatchAnyVersion() { + + var policyPatch = """ + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """; + var base64EncodedPolicy = new String( + Base64.getEncoder().encode(policyPatch.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8 + ); + + final Policy expectedPolicy = new Policy( + "test-plugin", + List.of( + new Scope("entitlement-module-name", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("entitlement-module-name-2", List.of(new SetHttpsConnectionPropertiesEntitlement())) + ) + ); + + var policy = PolicyUtils.parseEncodedPolicyIfExists( + base64EncodedPolicy, + "abcdef", + true, + "test-plugin", + Set.of("entitlement-module-name", "entitlement-module-name-2") + ); + + assertThat(policy, equalTo(expectedPolicy)); + } + + public void testNoPatchWithVersionMismatch() { + + var policyPatch = """ + versions: + - 9.0.0 + - 9.0.0-SNAPSHOT + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """; + var base64EncodedPolicy = new String( + Base64.getEncoder().encode(policyPatch.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8 + ); + + var policy = PolicyUtils.parseEncodedPolicyIfExists( + base64EncodedPolicy, + "9.1.0", + true, + "test-plugin", + Set.of("entitlement-module-name", "entitlement-module-name-2") + ); + + assertThat(policy, nullValue()); + } + + public void testNoPatchWithValidationError() { + + var policyPatch = """ + versions: + - 9.0.0 + - 9.0.0-SNAPSHOT + policy: + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """; + var base64EncodedPolicy = new String( + Base64.getEncoder().encode(policyPatch.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8 + ); + + var policy = PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of()); + + assertThat(policy, nullValue()); + } + + public void testNoPatchWithParsingError() { + + var policyPatch = """ + entitlement-module-name: + - load_native_libraries + entitlement-module-name-2: + - set_https_connection_properties + """; + var base64EncodedPolicy = new String( + Base64.getEncoder().encode(policyPatch.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8 + ); + + var policy = PolicyUtils.parseEncodedPolicyIfExists(base64EncodedPolicy, "9.0.0", true, "test-plugin", Set.of()); + + assertThat(policy, nullValue()); + } + + public void testMergeScopes() { + var originalPolicy = List.of( + new Scope("module1", List.of(new LoadNativeLibrariesEntitlement())), + new Scope("module2", List.of(new ManageThreadsEntitlement())), + new Scope("module3", List.of(new InboundNetworkEntitlement())) + ); + + var patchPolicy = List.of( + new Scope("module2", List.of(new ManageThreadsEntitlement())), + new Scope("module3", List.of(new OutboundNetworkEntitlement())), + new Scope("module4", List.of(new WriteAllSystemPropertiesEntitlement())) + ); + + var resultPolicy = PolicyUtils.mergeScopes(originalPolicy, patchPolicy); + assertThat( + resultPolicy, + containsInAnyOrder( + equalTo(new Scope("module1", List.of(new LoadNativeLibrariesEntitlement()))), + equalTo(new Scope("module2", List.of(new ManageThreadsEntitlement()))), + both(transformedMatch(Scope::moduleName, equalTo("module3"))).and( + transformedMatch( + Scope::entitlements, + containsInAnyOrder(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement()) + ) + ), + equalTo(new Scope("module4", List.of(new WriteAllSystemPropertiesEntitlement()))) + ) + ); + } + + public void testMergeSameFlagEntitlement() { + var e1 = new InboundNetworkEntitlement(); + var e2 = new InboundNetworkEntitlement(); + + assertThat(PolicyUtils.mergeEntitlement(e1, e2), equalTo(new InboundNetworkEntitlement())); + } + + public void testMergeFilesEntitlement() { + var e1 = new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/a/b"), FilesEntitlement.Mode.READ), + FilesEntitlement.FileData.ofPath(Path.of("/a/c"), FilesEntitlement.Mode.READ_WRITE), + FilesEntitlement.FileData.ofRelativePath(Path.of("c/d"), FilesEntitlement.BaseDir.CONFIG, FilesEntitlement.Mode.READ) + ) + ); + var e2 = new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/a/b"), FilesEntitlement.Mode.READ), // identical + FilesEntitlement.FileData.ofPath(Path.of("/a/c"), FilesEntitlement.Mode.READ), // different mode + FilesEntitlement.FileData.ofPath(Path.of("/c/d"), FilesEntitlement.Mode.READ) // different type + ) + ); + + var merged = PolicyUtils.mergeEntitlement(e1, e2); + assertThat( + merged, + transformedMatch( + x -> ((FilesEntitlement) x).filesData(), + containsInAnyOrder( + FilesEntitlement.FileData.ofPath(Path.of("/a/b"), FilesEntitlement.Mode.READ), + FilesEntitlement.FileData.ofPath(Path.of("/a/c"), FilesEntitlement.Mode.READ), + FilesEntitlement.FileData.ofPath(Path.of("/a/c"), FilesEntitlement.Mode.READ_WRITE), + FilesEntitlement.FileData.ofRelativePath(Path.of("c/d"), FilesEntitlement.BaseDir.CONFIG, FilesEntitlement.Mode.READ), + FilesEntitlement.FileData.ofPath(Path.of("/c/d"), FilesEntitlement.Mode.READ) + ) + ) + ); + } + + public void testMergeWritePropertyEntitlement() { + var e1 = new WriteSystemPropertiesEntitlement(List.of("a", "b", "c")); + var e2 = new WriteSystemPropertiesEntitlement(List.of("b", "c", "d")); + + var merged = PolicyUtils.mergeEntitlement(e1, e2); + assertThat( + merged, + transformedMatch(x -> ((WriteSystemPropertiesEntitlement) x).properties(), containsInAnyOrder("a", "b", "c", "d")) + ); + } + + public void testMergeEntitlements() { + List a = List.of( + new InboundNetworkEntitlement(), + new OutboundNetworkEntitlement(), + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/a/b"), FilesEntitlement.Mode.READ), + FilesEntitlement.FileData.ofPath(Path.of("/a/c"), FilesEntitlement.Mode.READ_WRITE) + ) + ) + ); + List b = List.of( + new InboundNetworkEntitlement(), + new LoadNativeLibrariesEntitlement(), + new FilesEntitlement(List.of()), + new WriteSystemPropertiesEntitlement(List.of("a")) + ); + + var merged = PolicyUtils.mergeEntitlements(a, b); + assertThat( + merged, + containsInAnyOrder( + new InboundNetworkEntitlement(), + new OutboundNetworkEntitlement(), + new LoadNativeLibrariesEntitlement(), + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(Path.of("/a/b"), FilesEntitlement.Mode.READ), + FilesEntitlement.FileData.ofPath(Path.of("/a/c"), FilesEntitlement.Mode.READ_WRITE) + ) + ), + new WriteSystemPropertiesEntitlement(List.of("a")) + ) + ); + } + + /** Test that we can parse the set of entitlements correctly for a simple policy */ + public void testFormatSimplePolicy() { + var pluginPolicy = new Policy( + "test-plugin", + List.of(new Scope("module1", List.of(new WriteSystemPropertiesEntitlement(List.of("property1", "property2"))))) + ); + + Set actual = PolicyUtils.getEntitlementsDescriptions(pluginPolicy); + assertThat(actual, containsInAnyOrder("write_system_properties [property1]", "write_system_properties [property2]")); + } + + /** Test that we can format the set of entitlements correctly for a complex policy */ + public void testFormatPolicyWithMultipleScopes() { + var pluginPolicy = new Policy( + "test-plugin", + List.of( + new Scope("module1", List.of(new CreateClassLoaderEntitlement())), + new Scope("module2", List.of(new CreateClassLoaderEntitlement(), new OutboundNetworkEntitlement())), + new Scope("module3", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())) + ) + ); + + Set actual = PolicyUtils.getEntitlementsDescriptions(pluginPolicy); + assertThat(actual, containsInAnyOrder("create_class_loader", "outbound_network", "inbound_network")); + } + + /** Test that we can format some simple files entitlement properly */ + public void testFormatFilesEntitlement() { + var pathAB = Path.of("/a/b"); + var pathCD = Path.of("c/d"); + var policy = new Policy( + "test-plugin", + List.of( + new Scope( + "module1", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(pathAB, FilesEntitlement.Mode.READ_WRITE), + FilesEntitlement.FileData.ofRelativePath(pathCD, FilesEntitlement.BaseDir.DATA, FilesEntitlement.Mode.READ) + ) + ) + ) + ), + new Scope( + "module2", + List.of( + new FilesEntitlement( + List.of( + FilesEntitlement.FileData.ofPath(pathAB, FilesEntitlement.Mode.READ_WRITE), + FilesEntitlement.FileData.ofPathSetting( + "setting", + FilesEntitlement.BaseDir.DATA, + FilesEntitlement.Mode.READ + ) + ) + ) + ) + ) + ) + ); + Set actual = PolicyUtils.getEntitlementsDescriptions(policy); + var pathABString = pathAB.toAbsolutePath().toString(); + var pathCDString = SEPARATOR + pathCD.toString(); + var pathSettingString = SEPARATOR + ""; + assertThat( + actual, + containsInAnyOrder( + "files [READ_WRITE] " + pathABString, + "files [READ] " + pathCDString, + "files [READ] " + pathSettingString + ) + ); + } + + /** Test that we can format some simple files entitlement properly */ + public void testFormatWriteSystemPropertiesEntitlement() { + var policy = new Policy( + "test-plugin", + List.of( + new Scope("module1", List.of(new WriteSystemPropertiesEntitlement(List.of("property1", "property2")))), + new Scope("module2", List.of(new WriteSystemPropertiesEntitlement(List.of("property2", "property3")))) + ) + ); + Set actual = PolicyUtils.getEntitlementsDescriptions(policy); + assertThat( + actual, + containsInAnyOrder( + "write_system_properties [property1]", + "write_system_properties [property2]", + "write_system_properties [property3]" + ) + ); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java new file mode 100644 index 0000000000000..7bc8e39fb1b27 --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy.entitlements; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.entitlement.runtime.policy.PathLookup; +import org.elasticsearch.entitlement.runtime.policy.Policy; +import org.elasticsearch.entitlement.runtime.policy.PolicyParser; +import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException; +import org.elasticsearch.entitlement.runtime.policy.Scope; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData; +import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.CONFIG; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ; +import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; + +public class FilesEntitlementTests extends ESTestCase { + + static Settings settings; + + @BeforeClass + public static void setupRoot() { + settings = Settings.EMPTY; + } + + private static final PathLookup TEST_PATH_LOOKUP = new PathLookup( + Path.of("home"), + Path.of("/config"), + new Path[] { Path.of("/data1"), Path.of("/data2") }, + new Path[] { Path.of("/shared1"), Path.of("/shared2") }, + Path.of("/tmp"), + pattern -> settings.getValues(pattern) + ); + + public void testEmptyBuild() { + PolicyValidationException pve = expectThrows(PolicyValidationException.class, () -> FilesEntitlement.build(List.of())); + assertEquals("must specify at least one path", pve.getMessage()); + pve = expectThrows(PolicyValidationException.class, () -> FilesEntitlement.build(null)); + assertEquals("must specify at least one path", pve.getMessage()); + } + + public void testInvalidRelativeDirectory() { + var ex = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build(List.of((Map.of("relative_path", "foo", "mode", "read", "relative_to", "bar")))) + ); + assertThat(ex.getMessage(), is("invalid relative directory: bar, valid values: [config, data, home]")); + } + + public void testFileDataRelativeWithAbsoluteDirectoryFails() { + var fileData = FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE); + var dataDirs = fileData.resolvePaths(TEST_PATH_LOOKUP); + assertThat(dataDirs.toList(), contains(Path.of("/data1/"), Path.of("/data2"))); + } + + public void testFileDataAbsoluteWithRelativeDirectoryFails() { + var ex = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build(List.of((Map.of("path", "foo", "mode", "read")))) + ); + + assertThat(ex.getMessage(), is("'path' [foo] must be absolute")); + } + + public void testFileDataRelativeWithEmptyDirectory() { + var ex = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build(List.of((Map.of("relative_path", "/foo", "mode", "read", "relative_to", "config")))) + ); + + var ex2 = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build(List.of((Map.of("relative_path", "C:\\foo", "mode", "read", "relative_to", "config")))) + ); + + assertThat(ex.getMessage(), is("'relative_path' [/foo] must be relative")); + assertThat(ex2.getMessage(), is("'relative_path' [C:\\foo] must be relative")); + } + + public void testPathSettingResolve() { + var entitlement = FilesEntitlement.build( + List.of(Map.of("path_setting", "foo.bar", "basedir_if_relative", "config", "mode", "read")) + ); + var filesData = entitlement.filesData(); + assertThat(filesData, contains(FileData.ofPathSetting("foo.bar", CONFIG, READ))); + + var fileData = FileData.ofPathSetting("foo.bar", CONFIG, READ); + // empty settings + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), empty()); + + fileData = FileData.ofPathSetting("foo.bar", CONFIG, READ); + settings = Settings.builder().put("foo.bar", "/setting/path").build(); + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path"))); + + fileData = FileData.ofPathSetting("foo.*.bar", CONFIG, READ); + settings = Settings.builder().put("foo.baz.bar", "/setting/path").build(); + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path"))); + + fileData = FileData.ofPathSetting("foo.*.bar", CONFIG, READ); + settings = Settings.builder().put("foo.baz.bar", "/setting/path").put("foo.baz2.bar", "/other/path").build(); + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), containsInAnyOrder(Path.of("/setting/path"), Path.of("/other/path"))); + + fileData = FileData.ofPathSetting("foo.bar", CONFIG, READ); + settings = Settings.builder().put("foo.bar", "relative_path").build(); + assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/config/relative_path"))); + } + + public void testPathSettingBasedirValidation() { + var e = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build(List.of(Map.of("path", "/foo", "mode", "read", "basedir_if_relative", "config"))) + ); + assertThat(e.getMessage(), is("'basedir_if_relative' may only be used with 'path_setting'")); + + e = expectThrows( + PolicyValidationException.class, + () -> FilesEntitlement.build( + List.of(Map.of("relative_path", "foo", "relative_to", "config", "mode", "read", "basedir_if_relative", "config")) + ) + ); + assertThat(e.getMessage(), is("'basedir_if_relative' may only be used with 'path_setting'")); + } + + public void testExclusiveParsing() throws Exception { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - files: + - path: /test + mode: read + exclusive: true + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of( + new Scope( + "entitlement-module-name", + List.of(FilesEntitlement.build(List.of(Map.of("path", "/test", "mode", "read", "exclusive", true)))) + ) + ) + ); + assertEquals(expected, parsedPolicy); + } +} diff --git a/libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml b/libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml deleted file mode 100644 index bbb926ccdd37d..0000000000000 --- a/libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml +++ /dev/null @@ -1,4 +0,0 @@ -entitlement-module-name: - - file: - path: "test/path/to/file" - mode: "read_write" diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java index 696be2808ed1f..f00db4f1e6601 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -116,6 +116,9 @@ public interface PointVisitor { boolean isValid(); Rectangle getResult(); + + /** To allow for memory optimizations through object reuse, the visitor can be reset to its initial state. */ + void reset(); } /** @@ -124,18 +127,14 @@ public interface PointVisitor { */ public static class CartesianPointVisitor implements PointVisitor { private double minX = Double.POSITIVE_INFINITY; - private double minY = Double.POSITIVE_INFINITY; private double maxX = Double.NEGATIVE_INFINITY; private double maxY = Double.NEGATIVE_INFINITY; + private double minY = Double.POSITIVE_INFINITY; public double getMinX() { return minX; } - public double getMinY() { - return minY; - } - public double getMaxX() { return maxX; } @@ -144,12 +143,16 @@ public double getMaxY() { return maxY; } + public double getMinY() { + return minY; + } + @Override public void visitPoint(double x, double y) { minX = Math.min(minX, x); - minY = Math.min(minY, y); maxX = Math.max(maxX, x); maxY = Math.max(maxY, y); + minY = Math.min(minY, y); } @Override @@ -160,9 +163,9 @@ public void visitRectangle(double minX, double maxX, double maxY, double minY) { ); } this.minX = Math.min(this.minX, minX); - this.minY = Math.min(this.minY, minY); this.maxX = Math.max(this.maxX, maxX); this.maxY = Math.max(this.maxY, maxY); + this.minY = Math.min(this.minY, minY); } @Override @@ -174,6 +177,14 @@ public boolean isValid() { public Rectangle getResult() { return new Rectangle(minX, maxX, maxY, minY); } + + @Override + public void reset() { + minX = Double.POSITIVE_INFINITY; + maxX = Double.NEGATIVE_INFINITY; + maxY = Double.NEGATIVE_INFINITY; + minY = Double.POSITIVE_INFINITY; + } } /** @@ -186,12 +197,12 @@ public Rectangle getResult() { * */ public static class GeoPointVisitor implements PointVisitor { - protected double minY = Double.POSITIVE_INFINITY; - protected double maxY = Double.NEGATIVE_INFINITY; - protected double minNegX = Double.POSITIVE_INFINITY; - protected double maxNegX = Double.NEGATIVE_INFINITY; - protected double minPosX = Double.POSITIVE_INFINITY; - protected double maxPosX = Double.NEGATIVE_INFINITY; + protected double top = Double.NEGATIVE_INFINITY; + protected double bottom = Double.POSITIVE_INFINITY; + protected double negLeft = Double.POSITIVE_INFINITY; + protected double negRight = Double.NEGATIVE_INFINITY; + protected double posLeft = Double.POSITIVE_INFINITY; + protected double posRight = Double.NEGATIVE_INFINITY; private final WrapLongitude wrapLongitude; @@ -199,69 +210,104 @@ public GeoPointVisitor(WrapLongitude wrapLongitude) { this.wrapLongitude = wrapLongitude; } + public double getTop() { + return top; + } + + public double getBottom() { + return bottom; + } + + public double getNegLeft() { + return negLeft; + } + + public double getNegRight() { + return negRight; + } + + public double getPosLeft() { + return posLeft; + } + + public double getPosRight() { + return posRight; + } + @Override public void visitPoint(double x, double y) { - minY = Math.min(minY, y); - maxY = Math.max(maxY, y); + bottom = Math.min(bottom, y); + top = Math.max(top, y); visitLongitude(x); } @Override public void visitRectangle(double minX, double maxX, double maxY, double minY) { - this.minY = Math.min(this.minY, minY); - this.maxY = Math.max(this.maxY, maxY); + // TODO: Fix bug with rectangle crossing the dateline (see Extent.addRectangle for correct behaviour) + this.bottom = Math.min(this.bottom, minY); + this.top = Math.max(this.top, maxY); visitLongitude(minX); visitLongitude(maxX); } private void visitLongitude(double x) { if (x >= 0) { - minPosX = Math.min(minPosX, x); - maxPosX = Math.max(maxPosX, x); + posLeft = Math.min(posLeft, x); + posRight = Math.max(posRight, x); } else { - minNegX = Math.min(minNegX, x); - maxNegX = Math.max(maxNegX, x); + negLeft = Math.min(negLeft, x); + negRight = Math.max(negRight, x); } } @Override public boolean isValid() { - return minY != Double.POSITIVE_INFINITY; + return bottom != Double.POSITIVE_INFINITY; } @Override public Rectangle getResult() { - return getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); + return getResult(top, bottom, negLeft, negRight, posLeft, posRight, wrapLongitude); + } + + @Override + public void reset() { + bottom = Double.POSITIVE_INFINITY; + top = Double.NEGATIVE_INFINITY; + negLeft = Double.POSITIVE_INFINITY; + negRight = Double.NEGATIVE_INFINITY; + posLeft = Double.POSITIVE_INFINITY; + posRight = Double.NEGATIVE_INFINITY; } - protected static Rectangle getResult( - double minNegX, - double minPosX, - double maxNegX, - double maxPosX, - double maxY, - double minY, + public static Rectangle getResult( + double top, + double bottom, + double negLeft, + double negRight, + double posLeft, + double posRight, WrapLongitude wrapLongitude ) { - assert Double.isFinite(maxY); - if (Double.isInfinite(minPosX)) { - return new Rectangle(minNegX, maxNegX, maxY, minY); - } else if (Double.isInfinite(minNegX)) { - return new Rectangle(minPosX, maxPosX, maxY, minY); + assert Double.isFinite(top); + if (posRight == Double.NEGATIVE_INFINITY) { + return new Rectangle(negLeft, negRight, top, bottom); + } else if (negLeft == Double.POSITIVE_INFINITY) { + return new Rectangle(posLeft, posRight, top, bottom); } else { return switch (wrapLongitude) { - case NO_WRAP -> new Rectangle(minNegX, maxPosX, maxY, minY); - case WRAP -> maybeWrap(minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + case NO_WRAP -> new Rectangle(negLeft, posRight, top, bottom); + case WRAP -> maybeWrap(top, bottom, negLeft, negRight, posLeft, posRight); }; } } - private static Rectangle maybeWrap(double minNegX, double minPosX, double maxNegX, double maxPosX, double maxY, double minY) { - double unwrappedWidth = maxPosX - minNegX; - double wrappedWidth = 360 + maxNegX - minPosX; + private static Rectangle maybeWrap(double top, double bottom, double negLeft, double negRight, double posLeft, double posRight) { + double unwrappedWidth = posRight - negLeft; + double wrappedWidth = 360 + negRight - posLeft; return unwrappedWidth <= wrappedWidth - ? new Rectangle(minNegX, maxPosX, maxY, minY) - : new Rectangle(minPosX, maxNegX, maxY, minY); + ? new Rectangle(negLeft, posRight, top, bottom) + : new Rectangle(posLeft, negRight, top, bottom); } } diff --git a/libs/h3/build.gradle b/libs/h3/build.gradle index 6036323e160fc..919050055b1ba 100644 --- a/libs/h3/build.gradle +++ b/libs/h3/build.gradle @@ -36,7 +36,7 @@ tasks.named('forbiddenApisMain').configure { } ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) -licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) +licenseFile.set(layout.settingsDirectory.file('licenses/APACHE-LICENSE-2.0.txt').asFile) tasks.withType(LicenseHeadersTask.class).configureEach { approvedLicenses = ['Apache', 'Generated', 'Vendored'] diff --git a/libs/logging/src/main/java/module-info.java b/libs/logging/src/main/java/module-info.java index 3ff21cefd14df..8c9ebbd91e869 100644 --- a/libs/logging/src/main/java/module-info.java +++ b/libs/logging/src/main/java/module-info.java @@ -9,5 +9,5 @@ module org.elasticsearch.logging { exports org.elasticsearch.logging; - exports org.elasticsearch.logging.internal.spi to org.elasticsearch.server; + exports org.elasticsearch.logging.internal.spi to org.elasticsearch.server, org.elasticsearch.cli; } diff --git a/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java b/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java index d5db919197307..a485a78a6e63f 100644 --- a/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java +++ b/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java @@ -9,6 +9,7 @@ package org.elasticsearch.logging.internal.spi; +import org.elasticsearch.logging.Level; import org.elasticsearch.logging.Logger; /** @@ -26,6 +27,10 @@ public static LoggerFactory provider() { public abstract Logger getLogger(Class clazz); + public abstract void setRootLevel(Level level); + + public abstract Level getRootLevel(); + public static void setInstance(LoggerFactory INSTANCE) { LoggerFactory.INSTANCE = INSTANCE; } diff --git a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java b/libs/native/src/main20/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java similarity index 82% rename from libs/native/src/main22/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java rename to libs/native/src/main20/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java index 34776407f759e..7bd359077e4f6 100644 --- a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java +++ b/libs/native/src/main20/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java @@ -11,7 +11,9 @@ public class NativeAccessUtil { /** - * Enables native access for the provided module. Available to JDK 22+, required for JDK 24+ when using --illegal-native-access=deny + * Enables native access for the provided module. + * Preview in JDK 20 and 21. + * Available to JDK 22+, required for JDK 24+ when using --illegal-native-access=deny */ public static void enableNativeAccess(ModuleLayer.Controller controller, Module module) { controller.enableNativeAccess(module); diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java index 5b7aab7ddfa48..758116300aa0f 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -36,7 +36,7 @@ public static ESVectorizationProvider getInstance() { static ESVectorizationProvider lookup(boolean testMode) { final int runtimeVersion = Runtime.version().feature(); assert runtimeVersion >= 21; - if (runtimeVersion <= 23) { + if (runtimeVersion <= 24) { // only use vector module with Hotspot VM if (Constants.IS_HOTSPOT_VM == false) { logger.warn("Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled."); diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle index d63df95003ab6..3ee86e206b582 100644 --- a/libs/ssl-config/build.gradle +++ b/libs/ssl-config/build.gradle @@ -10,6 +10,7 @@ apply plugin: "elasticsearch.publish" dependencies { api project(':libs:core') + api project(':libs:entitlement') testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'ssl-config' diff --git a/libs/ssl-config/src/main/java/module-info.java b/libs/ssl-config/src/main/java/module-info.java index 172ab6de193b3..16d86bbc6ced5 100644 --- a/libs/ssl-config/src/main/java/module-info.java +++ b/libs/ssl-config/src/main/java/module-info.java @@ -9,6 +9,7 @@ module org.elasticsearch.sslconfig { requires org.elasticsearch.base; + requires org.elasticsearch.entitlement; exports org.elasticsearch.common.ssl; } diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java index 6b708fa086cd0..069f042521902 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.PrivateKey; @@ -125,7 +124,7 @@ private PrivateKey getPrivateKey(Path path) { throw new SslConfigException("could not load ssl private key file [" + path + "]"); } return privateKey; - } catch (AccessControlException e) { + } catch (SecurityException e) { throw SslFileUtil.accessControlFailure(KEY_FILE_TYPE, List.of(path), e, configBasePath); } catch (IOException e) { throw SslFileUtil.ioException(KEY_FILE_TYPE, List.of(path), e); @@ -137,7 +136,7 @@ private PrivateKey getPrivateKey(Path path) { private List getCertificates(Path path) { try { return PemUtils.readCertificates(Collections.singleton(path)); - } catch (AccessControlException e) { + } catch (SecurityException e) { throw SslFileUtil.accessControlFailure(CERT_FILE_TYPE, List.of(path), e, configBasePath); } catch (IOException e) { throw SslFileUtil.ioException(CERT_FILE_TYPE, List.of(path), e); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java index 1c7295d6acc88..16aa02ef694d8 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; -import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.cert.Certificate; @@ -97,7 +96,7 @@ private Path resolveFile(String other) { private List readCertificates(List paths) { try { return PemUtils.readCertificates(paths); - } catch (AccessControlException e) { + } catch (SecurityException e) { throw SslFileUtil.accessControlFailure(CA_FILE_TYPE, paths, e, basePath); } catch (IOException e) { throw SslFileUtil.ioException(CA_FILE_TYPE, paths, e); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java index b7418a96f180c..8b11356d26fcd 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java @@ -18,7 +18,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.security.AccessControlException; import java.security.AlgorithmParameters; import java.security.GeneralSecurityException; import java.security.KeyFactory; @@ -110,7 +109,7 @@ public static PrivateKey readPrivateKey(Path path, Supplier passwordSupp throw new SslConfigException("could not load ssl private key file [" + path + "]"); } return privateKey; - } catch (AccessControlException e) { + } catch (SecurityException e) { throw SslFileUtil.accessControlFailure("PEM private key", List.of(path), e, null); } catch (IOException e) { throw SslFileUtil.ioException("PEM private key", List.of(path), e); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java index 79bfaec6a6d11..1a18acf9de55d 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java @@ -69,7 +69,7 @@ public abstract class SslConfigurationLoader { : Arrays.asList("TLSv1.2", "TLSv1.1") ); - private static final List JDK12_CIPHERS = List.of( + private static final List PRE_JDK24_CIPHERS = List.of( // TLSv1.3 cipher has PFS, AEAD, hardware support "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", @@ -118,7 +118,44 @@ public abstract class SslConfigurationLoader { "TLS_RSA_WITH_AES_128_CBC_SHA" ); - static final List DEFAULT_CIPHERS = JDK12_CIPHERS; + private static final List JDK24_CIPHERS = List.of( + // TLSv1.3 cipher has PFS, AEAD, hardware support + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", + + // TLSv1.3 cipher has PFS, AEAD + "TLS_CHACHA20_POLY1305_SHA256", + + // PFS, AEAD, hardware support + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + + // PFS, AEAD, hardware support + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + + // PFS, AEAD + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + + // PFS, hardware support + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + + // PFS, hardware support + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + + // PFS, hardware support + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + + // PFS, hardware support + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA" + ); + + static final List DEFAULT_CIPHERS = Runtime.version().feature() < 24 ? PRE_JDK24_CIPHERS : JDK24_CIPHERS; private static final char[] EMPTY_PASSWORD = new char[0]; public static final List GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS = List.of(X509Field.SAN_OTHERNAME_COMMONNAME); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java index b5eee13550493..e715b86d6cfb5 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java @@ -9,12 +9,13 @@ package org.elasticsearch.common.ssl; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; + import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.UnrecoverableKeyException; import java.util.List; @@ -78,7 +79,15 @@ static SslConfigException accessDenied(String fileType, List paths, Access return new SslConfigException(message, cause); } - static SslConfigException accessControlFailure(String fileType, List paths, AccessControlException cause, Path basePath) { + static SslConfigException notEntitledFailure(String fileType, List paths, NotEntitledException cause, Path basePath) { + return innerAccessControlFailure(fileType, paths, cause, basePath); + } + + static SslConfigException accessControlFailure(String fileType, List paths, SecurityException cause, Path basePath) { + return innerAccessControlFailure(fileType, paths, cause, basePath); + } + + private static SslConfigException innerAccessControlFailure(String fileType, List paths, Exception cause, Path basePath) { String message = "cannot read configured " + fileType + " [" + pathsToString(paths) + "] because "; if (paths.size() == 1) { message += "access to read the file is blocked"; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java index 7952b7a89ed9a..d1583297599d4 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java @@ -14,7 +14,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -166,7 +165,7 @@ private KeyStore processKeyStore(KeyStore keyStore) { private KeyStore readKeyStore(Path path) { try { return KeyStoreUtil.readKeyStore(path, type, storePassword); - } catch (AccessControlException e) { + } catch (SecurityException e) { throw SslFileUtil.accessControlFailure("[" + type + "] keystore", List.of(path), e, configBasePath); } catch (IOException e) { throw SslFileUtil.ioException("[" + type + "] keystore", List.of(path), e); diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java index 0d5c28e652f32..52850ba6a0030 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.security.AccessControlException; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.cert.X509Certificate; @@ -93,7 +92,7 @@ public X509ExtendedTrustManager createTrustManager() { private KeyStore readKeyStore(Path path) { try { return KeyStoreUtil.readKeyStore(path, type, password); - } catch (AccessControlException e) { + } catch (SecurityException e) { throw SslFileUtil.accessControlFailure(fileTypeForException(), List.of(path), e, configBasePath); } catch (IOException e) { throw SslFileUtil.ioException(fileTypeForException(), List.of(path), e, getAdditionalErrorDetails()); diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle index b79a6ce0a486a..47fc0dbc239cf 100644 --- a/libs/tdigest/build.gradle +++ b/libs/tdigest/build.gradle @@ -37,7 +37,7 @@ tasks.named('forbiddenApisMain').configure { } ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) -licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) +licenseFile.set(layout.settingsDirectory.file('licenses/APACHE-LICENSE-2.0.txt').asFile) tasks.withType(LicenseHeadersTask.class).configureEach { approvedLicenses = ['Apache', 'Generated', 'Vendored'] diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index 9faedd67974a9..c05f8482f7ef6 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -28,7 +28,7 @@ restResources { } } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java index d225ccc9d173f..a1119c64c577d 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java @@ -43,6 +43,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class SearchCancellationIT extends AbstractSearchCancellationTestCase { @@ -97,9 +98,7 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception { } logger.info("Executing search"); - // we have to explicitly set error_trace=true for the later exception check for `TimeSeriesIndexSearcher` Client client = client(); - client.threadPool().getThreadContext().putHeader("error_trace", "true"); TimeSeriesAggregationBuilder timeSeriesAggregationBuilder = new TimeSeriesAggregationBuilder("test_agg"); ActionFuture searchResponse = client.prepareSearch("test") .setQuery(matchAllQuery()) @@ -129,7 +128,9 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception { logger.info("All shards failed with", ex); if (lowLevelCancellation) { // Ensure that we cancelled in TimeSeriesIndexSearcher and not in reduce phase - assertThat(ExceptionsHelper.stackTrace(ex), containsString("TimeSeriesIndexSearcher")); + assertThat(ExceptionsHelper.stackTrace(ex), not(containsString("not building sub-aggregations due to task cancellation"))); + } else { + assertThat(ExceptionsHelper.stackTrace(ex), containsString("not building sub-aggregations due to task cancellation")); } } } diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java index 6add1b0ac4a13..abd482d8298ef 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java @@ -35,6 +35,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; import java.util.Collections; @@ -573,7 +574,15 @@ private void rebucket() { long[] mergeMap = new long[Math.toIntExact(oldOrds.size())]; bucketOrds = new LongKeyedBucketOrds.FromMany(bigArrays()); success = true; - for (long owningBucketOrd = 0; owningBucketOrd <= oldOrds.maxOwningBucketOrd(); owningBucketOrd++) { + long maxOwning = oldOrds.maxOwningBucketOrd(); + for (long owningBucketOrd = 0; owningBucketOrd <= maxOwning; owningBucketOrd++) { + /* + * Check for cancelation during this tight loop as it can take a while and the standard + * cancelation checks don't run during the loop. Becuase it's a tight loop. + */ + if (context.isCancelled()) { + throw new TaskCancelledException("cancelled"); + } LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = oldOrds.ordsEnum(owningBucketOrd); Rounding.Prepared preparedRounding = preparedRoundings[roundingIndexFor(owningBucketOrd)]; while (ordsEnum.next()) { diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java index 74c1f3c16278f..2eb21cfc09650 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java @@ -36,7 +36,7 @@ public void testNoData() throws Exception { MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg").fields( Collections.singletonList("field") ); - InternalMatrixStats stats = searchAndReduce(reader, new AggTestConfig(aggBuilder, ft)); + InternalMatrixStats stats = searchAndReduce(reader, new AggTestConfig(aggBuilder, ft).noReductionCancellation()); assertNull(stats.getStats()); assertEquals(0L, stats.getDocCount()); } @@ -54,7 +54,7 @@ public void testUnmapped() throws Exception { MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg").fields( Collections.singletonList("bogus") ); - InternalMatrixStats stats = searchAndReduce(reader, new AggTestConfig(aggBuilder, ft)); + InternalMatrixStats stats = searchAndReduce(reader, new AggTestConfig(aggBuilder, ft).noReductionCancellation()); assertNull(stats.getStats()); assertEquals(0L, stats.getDocCount()); } @@ -88,7 +88,7 @@ public void testTwoFields() throws Exception { MatrixStatsAggregationBuilder aggBuilder = new MatrixStatsAggregationBuilder("my_agg").fields( Arrays.asList(fieldA, fieldB) ); - InternalMatrixStats stats = searchAndReduce(reader, new AggTestConfig(aggBuilder, ftA, ftB)); + InternalMatrixStats stats = searchAndReduce(reader, new AggTestConfig(aggBuilder, ftA, ftB).noReductionCancellation()); multiPassStats.assertNearlyEqual(stats); assertTrue(MatrixAggregationInspectionHelper.hasValue(stats)); } diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java index 40a7b64bc91e7..a385db95d882a 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java @@ -207,7 +207,7 @@ public void testSynonymsInMultiplexerUpdateable() throws FileNotFoundException, public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundException, IOException { String synonymsFileName = "synonyms.txt"; setupResourceFile(synonymsFileName, "foo, baz"); - Path configDir = node().getEnvironment().configFile(); + Path configDir = node().getEnvironment().configDir(); if (Files.exists(configDir) == false) { Files.createDirectory(configDir); } @@ -319,7 +319,7 @@ public void testKeywordMarkerUpdateable() throws IOException { } private Path setupResourceFile(String fileName, String... content) throws IOException { - Path configDir = node().getEnvironment().configFile(); + Path configDir = node().getEnvironment().configDir(); if (Files.exists(configDir) == false) { Files.createDirectory(configDir); } diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java index 8209d9f543a31..06f19c0d60dba 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java @@ -57,7 +57,7 @@ public void testSynonymsWithPreview() throws FileNotFoundException, IOException, } private void testSynonymsUpdate(boolean preview) throws FileNotFoundException, IOException, InterruptedException { - Path config = internalCluster().getInstance(Environment.class).configFile(); + Path config = internalCluster().getInstance(Environment.class).configDir(); String synonymsFileName = "synonyms.txt"; Path synonymsFile = config.resolve(synonymsFileName); writeFile(synonymsFile, "foo, baz"); @@ -106,7 +106,7 @@ public void testSynonymsUpdateInvalid() throws IOException { final String synonymsFileName = "synonyms.txt"; final String fieldName = "field"; - Path config = internalCluster().getInstance(Environment.class).configFile(); + Path config = internalCluster().getInstance(Environment.class).configDir(); Path synonymsFile = config.resolve(synonymsFileName); writeFile(synonymsFile, "foo, baz"); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java index e091f0175009e..92e2b3085cc29 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java @@ -40,7 +40,7 @@ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundW throw new IllegalArgumentException("hyphenation_patterns_path is a required setting."); } - Path hyphenationPatternsFile = env.configFile().resolve(hyphenationPatternsPath); + Path hyphenationPatternsFile = env.configDir().resolve(hyphenationPatternsPath); try { InputStream in = Files.newInputStream(hyphenationPatternsFile); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java index 9e31fdde4330b..3dcbd87820f21 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java @@ -74,7 +74,7 @@ public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexC ); } else { reader = new ReaderWithOrigin( - Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService), + Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService, factory.lenient), "[" + synonymsSet + "] synonyms_set in .synonyms index", synonymsSet ); diff --git a/modules/analysis-common/src/main/plugin-metadata/entitlement-policy.yaml b/modules/analysis-common/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..e59b9dd27406c --- /dev/null +++ b/modules/analysis-common/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,5 @@ +org.elasticsearch.analysis.common: + - files: + - relative_path: analysis + relative_to: config + mode: read diff --git a/modules/apm/build.gradle b/modules/apm/build.gradle index de50d88a46d73..86d06258bcbca 100644 --- a/modules/apm/build.gradle +++ b/modules/apm/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-api:${otelVersion}" implementation "io.opentelemetry:opentelemetry-context:${otelVersion}" implementation "io.opentelemetry:opentelemetry-semconv:${otelSemconvVersion}" - runtimeOnly "co.elastic.apm:elastic-apm-agent:1.52.0" + runtimeOnly "co.elastic.apm:elastic-apm-agent:1.52.2" javaRestTestImplementation project(':modules:apm') javaRestTestImplementation project(':test:framework') diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index 9d4822aa9c4d6..68adc97b74449 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -90,6 +90,11 @@ public void initAgentSystemProperties(Settings settings) { */ @SuppressForbidden(reason = "Need to be able to manipulate APM agent-related properties to set them dynamically") public void setAgentSetting(String key, String value) { + if (key.startsWith("global_labels.")) { + // Invalid agent setting, leftover from flattening global labels in APMJVMOptions + // https://github.com/elastic/elasticsearch/issues/120791 + return; + } final String completeKey = "elastic.apm." + Objects.requireNonNull(key); AccessController.doPrivileged((PrivilegedAction) () -> { if (value == null || value.isEmpty()) { @@ -242,8 +247,8 @@ private static Setting concreteAgentSetting(String namespace, String qua return new Setting<>(qualifiedKey, "", (value) -> { if (qualifiedKey.equals("_na_") == false && PERMITTED_AGENT_KEYS.contains(namespace) == false) { if (namespace.startsWith("global_labels.")) { - // The nested labels syntax is transformed in APMJvmOptions. - // Ignore these here to not fail if not correctly removed. + // Invalid agent setting, leftover from flattening global labels in APMJVMOptions + // https://github.com/elastic/elasticsearch/issues/120791 return value; } throw new IllegalArgumentException("Configuration [" + qualifiedKey + "] is either prohibited or unknown."); diff --git a/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml index d80db1b99a1d1..216c67c492260 100644 --- a/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml @@ -83,9 +83,3 @@ org.elasticsearch.telemetry.apm: - elastic.apm.application_packages - elastic.apm.stack_trace_limit - elastic.apm.span_stack_trace_min_duration -elastic.apm.agent: - - set_https_connection_properties - - write_system_properties: - properties: - - AsyncProfiler.safemode - - load_native_libraries diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index de87e8bccb785..0c5a7a9fb42c7 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -36,7 +36,7 @@ if (buildParams.inFipsJvm){ tasks.named("yamlRestTest").configure{enabled = false } } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.withType(Test).configureEach { systemProperty 'es.failure_store_feature_flag_enabled', 'true' } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java index 8f410a10376fa..988fa8eab2ef7 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java @@ -323,6 +323,7 @@ public Collection getSystemDataStreamDescriptors() { .build(), Map.of(), List.of("product"), + "product", ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS ) ); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java index 2083807b1227f..d1f24d8d00f23 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java @@ -275,6 +275,7 @@ public Collection getSystemDataStreamDescriptors() { .build(), Map.of(), Collections.singletonList("test"), + "test", new ExecutorNames(ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, ThreadPool.Names.SYSTEM_WRITE) ) ); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java index dd3f1e74d4f4e..0e3847d4b7874 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java @@ -210,6 +210,7 @@ public Collection getSystemDataStreamDescriptors() { .build(), Map.of(), List.of("product"), + "product", ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS ) ); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 1c00e3ad380dc..f0827ebc8dfb8 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -104,6 +104,7 @@ import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -785,14 +786,10 @@ public void testErrorRecordingOnRetention() throws Exception { ).get(); DataStreamLifecycleHealthInfo dslHealthInfoOnHealthNode = healthNodeResponse.getHealthInfo().dslHealthInfo(); assertThat(dslHealthInfoOnHealthNode, is(not(DataStreamLifecycleHealthInfo.NO_DSL_ERRORS))); - // perhaps surprisingly rollover and delete are error-ing due to the read_only block on the first generation - // index which prevents metadata updates so rolling over the data stream is also blocked (note that both indices error at - // the same time so they'll have an equal retry count - the order becomes of the results, usually ordered by retry count, - // becomes non deterministic, hence the dynamic matching of index name) - assertThat(dslHealthInfoOnHealthNode.dslErrorsInfo().size(), is(2)); + assertThat(dslHealthInfoOnHealthNode.dslErrorsInfo().size(), is(1)); DslErrorInfo errorInfo = dslHealthInfoOnHealthNode.dslErrorsInfo().get(0); assertThat(errorInfo.retryCount(), greaterThanOrEqualTo(3)); - assertThat(List.of(firstGenerationIndex, secondGenerationIndex).contains(errorInfo.indexName()), is(true)); + assertThat(errorInfo.indexName(), equalTo(firstGenerationIndex)); }); GetHealthAction.Response healthResponse = client().execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(true, 1000)) @@ -808,15 +805,12 @@ public void testErrorRecordingOnRetention() throws Exception { assertThat(dslIndicator.impacts(), is(STAGNATING_INDEX_IMPACT)); assertThat( dslIndicator.symptom(), - is("2 backing indices have repeatedly encountered errors whilst trying to advance in its lifecycle") + is("A backing index has repeatedly encountered errors whilst trying to advance in its lifecycle") ); Diagnosis diagnosis = dslIndicator.diagnosisList().get(0); assertThat(diagnosis.definition(), is(STAGNATING_BACKING_INDICES_DIAGNOSIS_DEF)); - assertThat( - diagnosis.affectedResources().get(0).getValues(), - containsInAnyOrder(firstGenerationIndex, secondGenerationIndex) - ); + assertThat(diagnosis.affectedResources().get(0).getValues(), contains(firstGenerationIndex)); } // let's mark the index as writeable and make sure it's deleted and the error store is empty @@ -1278,6 +1272,7 @@ public Collection getSystemDataStreamDescriptors() { .build(), Map.of(), List.of(), + "test", ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS ) ); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index f60a3e5c47a7f..024f0af4ab9b2 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -28,6 +28,10 @@ public class DataStreamFeatures implements FeatureSpecification { public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); public static final NodeFeature DATA_STREAM_FAILURE_STORE_TSDB_FIX = new NodeFeature("data_stream.failure_store.tsdb_fix"); + public static final NodeFeature DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX = new NodeFeature( + "data_stream.downsample.default_aggregate_metric_fix" + ); + @Override public Map getHistoricalFeatures() { return Map.of(DATA_STREAM_LIFECYCLE, Version.V_8_11_0); @@ -45,6 +49,6 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX); + return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java index 44a64254f8da9..34262b334de14 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java @@ -107,14 +107,14 @@ ClusterState updateTimeSeriesTemporalRange(ClusterState current, Instant now) { // getWriteIndex() selects the latest added index: Index head = dataStream.getWriteIndex(); - IndexMetadata im = current.metadata().getIndexSafe(head); - Instant currentEnd = IndexSettings.TIME_SERIES_END_TIME.get(im.getSettings()); - TimeValue lookAheadTime = DataStreamsPlugin.getLookAheadTime(im.getSettings()); - Instant newEnd = DataStream.getCanonicalTimestampBound( - now.plus(lookAheadTime.getMillis(), ChronoUnit.MILLIS).plus(pollInterval.getMillis(), ChronoUnit.MILLIS) - ); - if (newEnd.isAfter(currentEnd)) { - try { + try { + IndexMetadata im = current.metadata().getIndexSafe(head); + Instant currentEnd = IndexSettings.TIME_SERIES_END_TIME.get(im.getSettings()); + TimeValue lookAheadTime = DataStreamsPlugin.getLookAheadTime(im.getSettings()); + Instant newEnd = DataStream.getCanonicalTimestampBound( + now.plus(lookAheadTime.getMillis(), ChronoUnit.MILLIS).plus(pollInterval.getMillis(), ChronoUnit.MILLIS) + ); + if (newEnd.isAfter(currentEnd)) { Settings settings = Settings.builder() .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DEFAULT_DATE_TIME_FORMATTER.format(newEnd)) .build(); @@ -131,17 +131,17 @@ ClusterState updateTimeSeriesTemporalRange(ClusterState current, Instant now) { mBuilder.updateSettings(settings, head.getName()); // Verify that all temporal ranges of each backing index is still valid: dataStream.validate(mBuilder::get); - } catch (Exception e) { - LOGGER.error( - () -> format( - "unable to update [%s] for data stream [%s] and backing index [%s]", - IndexSettings.TIME_SERIES_END_TIME.getKey(), - dataStream.getName(), - head.getName() - ), - e - ); } + } catch (Exception e) { + LOGGER.error( + () -> format( + "unable to update [%s] for data stream [%s] and backing index [%s]", + IndexSettings.TIME_SERIES_END_TIME.getKey(), + dataStream.getName(), + head.getName() + ), + e + ); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java index 262bc0b07960a..b5c031dfd75c9 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java @@ -20,21 +20,16 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; -import org.elasticsearch.snapshots.SnapshotInProgressException; -import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,6 +39,7 @@ import java.util.List; import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; import static org.elasticsearch.action.datastreams.DataStreamsActionUtil.getDataStreamNames; @@ -134,7 +130,6 @@ static ClusterState removeDataStream( for (String dataStreamName : dataStreams) { systemDataStreamAccessValidator.accept(dataStreamName); } - Set snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(currentState, dataStreams); if (dataStreams.isEmpty()) { if (request.isWildcardExpressionsOriginallySpecified()) { @@ -144,33 +139,11 @@ static ClusterState removeDataStream( } } - if (snapshottingDataStreams.isEmpty() == false) { - throw new SnapshotInProgressException( - "Cannot delete data streams that are being snapshotted: " - + snapshottingDataStreams - + ". Try again after snapshot finishes or cancel the currently running snapshot." - ); - } - - Set backingIndicesToRemove = new HashSet<>(); - for (String dataStreamName : dataStreams) { - DataStream dataStream = currentState.metadata().dataStreams().get(dataStreamName); - assert dataStream != null; - backingIndicesToRemove.addAll(dataStream.getIndices()); - backingIndicesToRemove.addAll(dataStream.getFailureIndices()); - } - - // first delete the data streams and then the indices: - // (this to avoid data stream validation from failing when deleting an index that is part of a data stream - // without updating the data stream) - // TODO: change order when delete index api also updates the data stream the index to be removed is member of - Metadata.Builder metadata = Metadata.builder(currentState.metadata()); - for (String ds : dataStreams) { - LOGGER.info("removing data stream [{}]", ds); - metadata.removeDataStream(ds); - } - currentState = ClusterState.builder(currentState).metadata(metadata).build(); - return MetadataDeleteIndexService.deleteIndices(currentState, backingIndicesToRemove, settings); + return MetadataDataStreamsService.deleteDataStreams( + currentState, + dataStreams.stream().map(currentState.metadata().dataStreams()::get).collect(Collectors.toSet()), + settings + ); } @Override diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 3d08be1f24a42..c150f64e8cc4a 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -248,7 +248,7 @@ public void setup() throws Exception { MetadataCreateIndexService createIndexService; { Environment env = mock(Environment.class); - when(env.sharedDataFile()).thenReturn(null); + when(env.sharedDataDir()).thenReturn(null); AllocationService allocationService = mock(AllocationService.class); when(allocationService.reroute(any(ClusterState.class), any(String.class), any())).then(i -> i.getArguments()[0]); when(allocationService.getShardRoutingRoleStrategy()).thenReturn(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java index 8378526e6bdae..9b069b31a50ce 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java @@ -8,11 +8,18 @@ */ package org.elasticsearch.datastreams; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.filter.RegexFilter; +import org.apache.logging.log4j.message.Message; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -22,15 +29,22 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAmount; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -41,6 +55,22 @@ public class UpdateTimeSeriesRangeServiceTests extends ESTestCase { + static MockAppender appender; + static Logger testLogger1 = LogManager.getLogger(UpdateTimeSeriesRangeService.class); + + @BeforeClass + public static void classInit() throws IllegalAccessException { + appender = new MockAppender("mock_appender"); + appender.start(); + Loggers.addAppender(testLogger1, appender); + } + + @AfterClass + public static void classCleanup() { + Loggers.removeAppender(testLogger1, appender); + appender.stop(); + } + private ThreadPool threadPool; private UpdateTimeSeriesRangeService instance; @@ -191,6 +221,68 @@ public void testUpdateTimeSeriesTemporalRangeMultipleDataStream() { assertThat(getEndTime(result, dataStreamName3, 0), equalTo(start)); } + public void testUpdateTimeSeriesTemporalOneBadDataStream() { + String dataStreamName1 = "logs-app1"; + String dataStreamName2 = "logs-app2-broken"; + String dataStreamName3 = "logs-app3"; + Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); + + Instant start = now.minus(90, ChronoUnit.MINUTES); + Instant end = start.plus(30, ChronoUnit.MINUTES); + Metadata.Builder mbBuilder = new Metadata.Builder(); + for (String dataStreamName : List.of(dataStreamName1, dataStreamName2, dataStreamName3)) { + DataStreamTestHelper.getClusterStateWithDataStream(mbBuilder, dataStreamName, List.of(new Tuple<>(start, end))); + } + + Settings settings = Settings.builder().put("index.mode", "logsdb").build(); + var im = createIndexMetadata(getDefaultBackingIndexName(dataStreamName2, 2, start.toEpochMilli()), true, settings, 0); + mbBuilder.put(im, true); + var ds2 = mbBuilder.dataStreamMetadata().dataStreams().get(dataStreamName2); + var ds2Indices = new ArrayList<>(ds2.getIndices()); + ds2Indices.add(im.getIndex()); + var copy = new HashMap<>(mbBuilder.dataStreamMetadata().dataStreams()); + copy.put( + dataStreamName2, + new DataStream( + ds2.getName(), + ds2Indices, + 2, + ds2.getMetadata(), + ds2.isHidden(), + ds2.isReplicated(), + ds2.isSystem(), + ds2.isAllowCustomRouting(), + ds2.getIndexMode(), + ds2.getLifecycle(), + ds2.getDataStreamOptions(), + ds2.getFailureIndices(), + ds2.rolloverOnWrite(), + ds2.getAutoShardingEvent() + ) + ); + mbBuilder.dataStreams(copy, Map.of()); + + now = now.minus(45, ChronoUnit.MINUTES); + ClusterState before = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(mbBuilder).build(); + ClusterState result = instance.updateTimeSeriesTemporalRange(before, now); + assertThat(result, not(sameInstance(before))); + final var expectedEndTime = now.plus(35, ChronoUnit.MINUTES).truncatedTo(ChronoUnit.SECONDS); + assertThat(getEndTime(result, dataStreamName1, 0), equalTo(expectedEndTime)); + assertThat(getEndTime(result, dataStreamName2, 0), equalTo(end)); // failed to update end_time, because broken data stream + assertThat(getEndTime(result, dataStreamName3, 0), equalTo(expectedEndTime)); + + String message = appender.getLastEventAndReset().getMessage().getFormattedMessage(); + assertThat( + message, + equalTo( + "unable to update [index.time_series.end_time] for data stream [logs-app2-broken] and " + + "backing index [" + + im.getIndex().getName() + + "]" + ) + ); + } + public void testUpdatePollInterval() { instance.scheduleTask(); assertThat(instance.pollInterval, equalTo(TimeValue.timeValueMinutes(5))); @@ -220,4 +312,27 @@ static Instant getStartTime(ClusterState state, String dataStreamName, int index return IndexSettings.TIME_SERIES_START_TIME.get(indexSettings); } + static class MockAppender extends AbstractAppender { + public LogEvent lastEvent; + + MockAppender(final String name) throws IllegalAccessException { + super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null, false); + } + + @Override + public void append(LogEvent event) { + lastEvent = event.toImmutable(); + } + + Message lastMessage() { + return lastEvent.getMessage(); + } + + public LogEvent getLastEventAndReset() { + LogEvent toReturn = lastEvent; + lastEvent = null; + return toReturn; + } + } + } diff --git a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java index 22f2a9fa394fb..68c6a5c826b34 100644 --- a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java +++ b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -49,7 +49,8 @@ private static ElasticsearchCluster createCluster() { .feature(FAILURE_STORE_ENABLED) .setting("xpack.security.enabled", "true") .keystore("bootstrap.password", "x-pack-test-password") - .user("x_pack_rest_user", "x-pack-test-password"); + .user("x_pack_rest_user", "x-pack-test-password") + .systemProperty("es.queryable_built_in_roles_enabled", "false"); if (initTestSeed().nextBoolean()) { clusterBuilder.setting("xpack.license.self_generated.type", "trial"); } diff --git a/modules/ingest-attachment/src/main/config/log4j2.properties b/modules/ingest-attachment/src/main/config/log4j2.properties index 92951ec9e0e56..0f1deb9bc48f8 100644 --- a/modules/ingest-attachment/src/main/config/log4j2.properties +++ b/modules/ingest-attachment/src/main/config/log4j2.properties @@ -9,3 +9,6 @@ logger.org_apache_fontbox.level = off logger.org_apache_xmlbeans.name = org.apache.xmlbeans logger.org_apache_xmlbeans.level = off + +logger.entitlements_ingest_attachment.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.ingest-attachment.ALL-UNNAMED +logger.entitlements_ingest_attachment.level = error diff --git a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java index 02d85ef0ecfbf..13447b620d14b 100644 --- a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java +++ b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java @@ -122,15 +122,20 @@ static String parse(final byte content[], final Metadata metadata, final int lim // apply additional containment for parsers, this is intersected with the current permissions // its hairy, but worth it so we don't have some XML flaw reading random crap from the FS - private static final AccessControlContext RESTRICTED_CONTEXT = new AccessControlContext( - new ProtectionDomain[] { new ProtectionDomain(null, getRestrictedPermissions()) } - ); + private static final AccessControlContext RESTRICTED_CONTEXT = isUsingSecurityManager() + ? new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, getRestrictedPermissions()) }) + : null; + + private static boolean isUsingSecurityManager() { + return false; + } // compute some minimal permissions for parsers. they only get r/w access to the java temp directory, // the ability to load some resources from JARs, and read sysprops @SuppressForbidden(reason = "adds access to tmp directory") static PermissionCollection getRestrictedPermissions() { Permissions perms = new Permissions(); + // property/env access needed for parsing perms.add(new PropertyPermission("*", "read")); perms.add(new RuntimePermission("getenv.TIKA_CONFIG")); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 22db5a330fb45..12901a1b4a938 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -93,17 +93,9 @@ public final class DateProcessor extends AbstractProcessor { formatter = DateFormatter.forPattern(this.outputFormat); } - private static ZoneId newDateTimeZone(String timezone) { - return timezone == null ? ZoneOffset.UTC : ZoneId.of(timezone); - } - - private static Locale newLocale(String locale) { - return locale == null ? Locale.ENGLISH : LocaleUtils.parse(locale); - } - @Override - public IngestDocument execute(IngestDocument ingestDocument) { - Object obj = ingestDocument.getFieldValue(field, Object.class); + public IngestDocument execute(IngestDocument document) { + Object obj = document.getFieldValue(field, Object.class); String value = null; if (obj != null) { // Don't use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers @@ -114,10 +106,9 @@ public IngestDocument execute(IngestDocument ingestDocument) { // extract the timezone and locale to use for date parsing final ZoneId documentTimezone; final Locale documentLocale; - final Map sourceAndMetadata = ingestDocument.getSourceAndMetadata(); try { - documentTimezone = newDateTimeZone(timezone == null ? null : timezone.newInstance(sourceAndMetadata).execute()); - documentLocale = newLocale(locale == null ? null : locale.newInstance(sourceAndMetadata).execute()); + documentTimezone = getTimezone(document); + documentLocale = getLocale(document); } catch (Exception e) { throw new IllegalArgumentException("unable to parse date [" + value + "]", e); } @@ -138,8 +129,8 @@ public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException); } - ingestDocument.setFieldValue(targetField, formatter.format(dateTime)); - return ingestDocument; + document.setFieldValue(targetField, formatter.format(dateTime)); + return document; } @Override @@ -147,12 +138,24 @@ public String getType() { return TYPE; } - TemplateScript.Factory getTimezone() { - return timezone; + // visible for testing + ZoneId getTimezone(IngestDocument document) { + String value = timezone == null ? null : document.renderTemplate(timezone); + if (value == null) { + return ZoneOffset.UTC; + } else { + return ZoneId.of(value); + } } - TemplateScript.Factory getLocale() { - return locale; + // visible for testing + Locale getLocale(IngestDocument document) { + String value = locale == null ? null : document.renderTemplate(locale); + if (value == null) { + return Locale.ENGLISH; + } else { + return LocaleUtils.parse(value); + } } String getField() { @@ -179,32 +182,22 @@ public Factory(ScriptService scriptService) { this.scriptService = scriptService; } - public DateProcessor create( - Map registry, - String processorTag, - String description, - Map config - ) throws Exception { - String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); - String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD); - String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone"); + public DateProcessor create(Map registry, String tag, String description, Map config) + throws Exception { + String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field", DEFAULT_TARGET_FIELD); + String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone"); TemplateScript.Factory compiledTimezoneTemplate = null; if (timezoneString != null) { - compiledTimezoneTemplate = ConfigurationUtils.compileTemplate( - TYPE, - processorTag, - "timezone", - timezoneString, - scriptService - ); + compiledTimezoneTemplate = ConfigurationUtils.compileTemplate(TYPE, tag, "timezone", timezoneString, scriptService); } - String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "locale"); + String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale"); TemplateScript.Factory compiledLocaleTemplate = null; if (localeString != null) { - compiledLocaleTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "locale", localeString, scriptService); + compiledLocaleTemplate = ConfigurationUtils.compileTemplate(TYPE, tag, "locale", localeString, scriptService); } - List formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats"); - String outputFormat = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT); + List formats = ConfigurationUtils.readList(TYPE, tag, config, "formats"); + String outputFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "output_format", DEFAULT_OUTPUT_FORMAT); try { DateFormatter.forPattern(outputFormat); } catch (Exception e) { @@ -212,7 +205,7 @@ public DateProcessor create( } return new DateProcessor( - processorTag, + tag, description, compiledTimezoneTemplate, compiledLocaleTemplate, diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index 97e26a9961c20..3b212c71f2220 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -11,6 +11,8 @@ import org.apache.http.conn.util.PublicSuffixMatcher; import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -19,9 +21,9 @@ import java.util.Map; public class RegisteredDomainProcessor extends AbstractProcessor { - private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); public static final String TYPE = "registered_domain"; + private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); private final String field; private final String targetField; @@ -47,17 +49,18 @@ public boolean getIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) throws Exception { - DomainInfo info = getRegisteredDomain(ingestDocument); + public IngestDocument execute(IngestDocument document) throws Exception { + final String fqdn = document.getFieldValue(field, String.class, ignoreMissing); + final DomainInfo info = getRegisteredDomain(fqdn); if (info == null) { if (ignoreMissing) { - return ingestDocument; + return document; } else { throw new IllegalArgumentException("unable to set domain information for document"); } } String fieldPrefix = targetField; - if (fieldPrefix.equals("") == false) { + if (fieldPrefix.isEmpty() == false) { fieldPrefix += "."; } String domainTarget = fieldPrefix + "domain"; @@ -65,30 +68,31 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String subdomainTarget = fieldPrefix + "subdomain"; String topLevelDomainTarget = fieldPrefix + "top_level_domain"; - if (info.getDomain() != null) { - ingestDocument.setFieldValue(domainTarget, info.getDomain()); + if (info.domain() != null) { + document.setFieldValue(domainTarget, info.domain()); } - if (info.getRegisteredDomain() != null) { - ingestDocument.setFieldValue(registeredDomainTarget, info.getRegisteredDomain()); + if (info.registeredDomain() != null) { + document.setFieldValue(registeredDomainTarget, info.registeredDomain()); } - if (info.getETLD() != null) { - ingestDocument.setFieldValue(topLevelDomainTarget, info.getETLD()); + if (info.eTLD() != null) { + document.setFieldValue(topLevelDomainTarget, info.eTLD()); } - if (info.getSubdomain() != null) { - ingestDocument.setFieldValue(subdomainTarget, info.getSubdomain()); + if (info.subdomain() != null) { + document.setFieldValue(subdomainTarget, info.subdomain()); } - return ingestDocument; + return document; } - private DomainInfo getRegisteredDomain(IngestDocument d) { - String fieldString = d.getFieldValue(field, String.class, ignoreMissing); - if (fieldString == null) { + @Nullable + // visible for testing + static DomainInfo getRegisteredDomain(@Nullable String fqdn) { + if (Strings.hasText(fqdn) == false) { return null; } - String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fieldString); + String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fqdn); if (registeredDomain == null) { - if (SUFFIX_MATCHER.matches(fieldString)) { - return new DomainInfo(fieldString); + if (SUFFIX_MATCHER.matches(fqdn)) { + return DomainInfo.of(fqdn); } return null; } @@ -96,7 +100,7 @@ private DomainInfo getRegisteredDomain(IngestDocument d) { // we have domain with no matching public suffix, but "." in it return null; } - return new DomainInfo(registeredDomain, fieldString); + return DomainInfo.of(registeredDomain, fqdn); } @Override @@ -104,54 +108,27 @@ public String getType() { return TYPE; } - private static class DomainInfo { - private final String domain; - private final String registeredDomain; - private final String eTLD; - private final String subdomain; - - private DomainInfo(String eTLD) { - this.domain = eTLD; - this.eTLD = eTLD; - this.registeredDomain = null; - this.subdomain = null; + // visible for testing + record DomainInfo( + String domain, + String registeredDomain, + String eTLD, // n.b. https://developer.mozilla.org/en-US/docs/Glossary/eTLD + String subdomain + ) { + static DomainInfo of(final String eTLD) { + return new DomainInfo(eTLD, null, eTLD, null); } - private DomainInfo(String registeredDomain, String domain) { + static DomainInfo of(final String registeredDomain, final String domain) { int index = registeredDomain.indexOf('.') + 1; if (index > 0 && index < registeredDomain.length()) { - this.domain = domain; - this.eTLD = registeredDomain.substring(index); - this.registeredDomain = registeredDomain; int subdomainIndex = domain.lastIndexOf("." + registeredDomain); - if (subdomainIndex > 0) { - this.subdomain = domain.substring(0, subdomainIndex); - } else { - this.subdomain = null; - } + final String subdomain = subdomainIndex > 0 ? domain.substring(0, subdomainIndex) : null; + return new DomainInfo(domain, registeredDomain, registeredDomain.substring(index), subdomain); } else { - this.domain = null; - this.eTLD = null; - this.registeredDomain = null; - this.subdomain = null; + return new DomainInfo(null, null, null, null); } } - - public String getDomain() { - return domain; - } - - public String getSubdomain() { - return subdomain; - } - - public String getRegisteredDomain() { - return registeredDomain; - } - - public String getETLD() { - return eTLD; - } } public static final class Factory implements Processor.Factory { @@ -161,15 +138,15 @@ public static final class Factory implements Processor.Factory { @Override public RegisteredDomainProcessor create( Map registry, - String processorTag, + String tag, String description, Map config ) throws Exception { - String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); - String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD); - boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true); + String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field", DEFAULT_TARGET_FIELD); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", true); - return new RegisteredDomainProcessor(processorTag, description, field, targetField, ignoreMissing); + return new RegisteredDomainProcessor(tag, description, field, targetField, ignoreMissing); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java index 76a219ec4458d..4b2193eb3c192 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java @@ -10,11 +10,14 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -43,8 +46,8 @@ public void testBuildDefaults() throws Exception { assertThat(processor.getField(), equalTo(sourceField)); assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD)); assertThat(processor.getFormats(), equalTo(List.of("dd/MM/yyyyy"))); - assertNull(processor.getLocale()); - assertNull(processor.getTimezone()); + assertThat(processor.getTimezone(null), equalTo(ZoneOffset.UTC)); + assertThat(processor.getLocale(null), equalTo(Locale.ENGLISH)); } public void testMatchFieldIsMandatory() throws Exception { @@ -81,11 +84,12 @@ public void testParseLocale() throws Exception { String sourceField = randomAlphaOfLengthBetween(1, 10); config.put("field", sourceField); config.put("formats", List.of("dd/MM/yyyyy")); - Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.ROOT); + Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.CANADA); config.put("locale", locale.toLanguageTag()); DateProcessor processor = factory.create(null, null, null, config); - assertThat(processor.getLocale().newInstance(Map.of()).execute(), equalTo(locale.toLanguageTag())); + IngestDocument document = RandomDocumentPicks.randomIngestDocument(random(), Map.of()); + assertThat(processor.getLocale(document), equalTo(locale)); } public void testParseTimezone() throws Exception { @@ -97,7 +101,8 @@ public void testParseTimezone() throws Exception { ZoneId timezone = randomZone(); config.put("timezone", timezone.getId()); DateProcessor processor = factory.create(null, null, null, config); - assertThat(processor.getTimezone().newInstance(Map.of()).execute(), equalTo(timezone.getId())); + IngestDocument document = RandomDocumentPicks.randomIngestDocument(random(), Map.of()); + assertThat(processor.getTimezone(document), equalTo(timezone)); } public void testParseMatchFormats() throws Exception { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java index 0a0666de9b014..b9fe870af2385 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java @@ -11,132 +11,172 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.TestIngestDocument; +import org.elasticsearch.ingest.common.RegisteredDomainProcessor.DomainInfo; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; import java.util.Map; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; +import static java.util.Map.entry; +import static org.elasticsearch.ingest.common.RegisteredDomainProcessor.getRegisteredDomain; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; /** * Test parsing of an eTLD from a FQDN. The list of eTLDs is maintained here: * https://github.com/publicsuffix/list/blob/master/public_suffix_list.dat - * - * Effective TLDs (eTLS) are not the same as DNS TLDs. Uses for eTLDs are listed here. + *

+ * Effective TLDs (eTLDs) are not the same as DNS TLDs. Uses for eTLDs are listed here: * https://publicsuffix.org/learn/ */ public class RegisteredDomainProcessorTests extends ESTestCase { - private Map buildEvent(String domain) { - return Map.of("domain", domain); - } - public void testBasic() throws Exception { - testRegisteredDomainProcessor(buildEvent("www.google.com"), "www.google.com", "google.com", "com", "www"); - testRegisteredDomainProcessor(buildEvent("google.com"), "google.com", "google.com", "com", null); - testRegisteredDomainProcessor(buildEvent(""), null, null, null, null); - testRegisteredDomainProcessor(buildEvent("."), null, null, null, null); - testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null); - testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null); - testRegisteredDomainProcessor(buildEvent("www.books.amazon.co.uk"), "www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books"); + public void testGetRegisteredDomain() { + assertThat(getRegisteredDomain("www.google.com"), is(new DomainInfo("www.google.com", "google.com", "com", "www"))); + assertThat(getRegisteredDomain("google.com"), is(new DomainInfo("google.com", "google.com", "com", null))); + assertThat(getRegisteredDomain(null), nullValue()); + assertThat(getRegisteredDomain(""), nullValue()); + assertThat(getRegisteredDomain(" "), nullValue()); + assertThat(getRegisteredDomain("."), nullValue()); + assertThat(getRegisteredDomain("$"), nullValue()); + assertThat(getRegisteredDomain("foo.bar.baz"), nullValue()); + assertThat( + getRegisteredDomain("www.books.amazon.co.uk"), + is(new DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) + ); // Verify "com" is returned as the eTLD, for that FQDN or subdomain - testRegisteredDomainProcessor(buildEvent("com"), "com", null, "com", null); - testRegisteredDomainProcessor(buildEvent("example.com"), "example.com", "example.com", "com", null); - testRegisteredDomainProcessor(buildEvent("googleapis.com"), "googleapis.com", "googleapis.com", "com", null); - testRegisteredDomainProcessor( - buildEvent("content-autofill.googleapis.com"), - "content-autofill.googleapis.com", - "googleapis.com", - "com", - "content-autofill" + assertThat(getRegisteredDomain("com"), is(new DomainInfo("com", null, "com", null))); + assertThat(getRegisteredDomain("example.com"), is(new DomainInfo("example.com", "example.com", "com", null))); + assertThat(getRegisteredDomain("googleapis.com"), is(new DomainInfo("googleapis.com", "googleapis.com", "com", null))); + assertThat( + getRegisteredDomain("content-autofill.googleapis.com"), + is(new DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) ); // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain - testRegisteredDomainProcessor( - buildEvent("global.ssl.fastly.net"), - "global.ssl.fastly.net", - "global.ssl.fastly.net", - "ssl.fastly.net", - null + assertThat( + getRegisteredDomain("global.ssl.fastly.net"), + is(new DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) ); - testRegisteredDomainProcessor( - buildEvent("1.www.global.ssl.fastly.net"), - "1.www.global.ssl.fastly.net", - "global.ssl.fastly.net", - "ssl.fastly.net", - "1.www" + assertThat( + getRegisteredDomain("1.www.global.ssl.fastly.net"), + is(new DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) ); } - public void testUseRoot() throws Exception { - Map source = buildEvent("www.google.co.uk"); - - String domainField = "domain"; - String registeredDomainField = "registered_domain"; - String topLevelDomainField = "top_level_domain"; - String subdomainField = "subdomain"; + public void testBasic() throws Exception { + var processor = new RegisteredDomainProcessor(null, null, "input", "output", false); + { + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("input", "www.google.co.uk")); + processor.execute(document); + assertThat( + document.getSource(), + is( + Map.ofEntries( + entry("input", "www.google.co.uk"), + entry( + "output", + Map.ofEntries( + entry("domain", "www.google.co.uk"), + entry("registered_domain", "google.co.uk"), + entry("top_level_domain", "co.uk"), + entry("subdomain", "www") + ) + ) + ) + ) + ); + } + { + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("input", "example.com")); + processor.execute(document); + assertThat( + document.getSource(), + is( + Map.ofEntries( + entry("input", "example.com"), + entry( + "output", + Map.ofEntries( + entry("domain", "example.com"), + entry("registered_domain", "example.com"), + entry("top_level_domain", "com") + ) + ) + ) + ) + ); + } + { + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("input", "com")); + processor.execute(document); + assertThat( + document.getSource(), + is( + Map.ofEntries( + entry("input", "com"), + entry( + "output", + Map.ofEntries( + entry("domain", "com"), // + entry("top_level_domain", "com") + ) + ) + ) + ) + ); + } + } + public void testUseRoot() throws Exception { var processor = new RegisteredDomainProcessor(null, null, "domain", "", false); - - IngestDocument input = TestIngestDocument.withDefaultVersion(source); - IngestDocument output = processor.execute(input); - - String domain = output.getFieldValue(domainField, String.class); - assertThat(domain, equalTo("www.google.co.uk")); - String registeredDomain = output.getFieldValue(registeredDomainField, String.class); - assertThat(registeredDomain, equalTo("google.co.uk")); - String eTLD = output.getFieldValue(topLevelDomainField, String.class); - assertThat(eTLD, equalTo("co.uk")); - String subdomain = output.getFieldValue(subdomainField, String.class); - assertThat(subdomain, equalTo("www")); + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "www.google.co.uk")); + processor.execute(document); + assertThat( + document.getSource(), + is( + Map.ofEntries( + entry("domain", "www.google.co.uk"), + entry("registered_domain", "google.co.uk"), + entry("top_level_domain", "co.uk"), + entry("subdomain", "www") + ) + ) + ); } public void testError() throws Exception { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null, false) - ); - assertThat(e.getMessage(), containsString("unable to set domain information for document")); - e = expectThrows( - IllegalArgumentException.class, - () -> testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null, false) - ); - assertThat(e.getMessage(), containsString("unable to set domain information for document")); - } + var processor = new RegisteredDomainProcessor(null, null, "domain", "", false); - private void testRegisteredDomainProcessor( - Map source, - String expectedDomain, - String expectedRegisteredDomain, - String expectedETLD, - String expectedSubdomain - ) throws Exception { - testRegisteredDomainProcessor(source, expectedDomain, expectedRegisteredDomain, expectedETLD, expectedSubdomain, true); + { + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "foo.bar.baz")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), is("unable to set domain information for document")); + assertThat(document.getSource(), is(Map.of("domain", "foo.bar.baz"))); + } + + { + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "$")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), is("unable to set domain information for document")); + assertThat(document.getSource(), is(Map.of("domain", "$"))); + } } - private void testRegisteredDomainProcessor( - Map source, - String expectedDomain, - String expectedRegisteredDomain, - String expectedETLD, - String expectedSubdomain, - boolean ignoreMissing - ) throws Exception { - String domainField = "url.domain"; - String registeredDomainField = "url.registered_domain"; - String topLevelDomainField = "url.top_level_domain"; - String subdomainField = "url.subdomain"; - - var processor = new RegisteredDomainProcessor(null, null, "domain", "url", ignoreMissing); - - IngestDocument input = TestIngestDocument.withDefaultVersion(source); - IngestDocument output = processor.execute(input); - - String domain = output.getFieldValue(domainField, String.class, expectedDomain == null); - assertThat(domain, equalTo(expectedDomain)); - String registeredDomain = output.getFieldValue(registeredDomainField, String.class, expectedRegisteredDomain == null); - assertThat(registeredDomain, equalTo(expectedRegisteredDomain)); - String eTLD = output.getFieldValue(topLevelDomainField, String.class, expectedETLD == null); - assertThat(eTLD, equalTo(expectedETLD)); - String subdomain = output.getFieldValue(subdomainField, String.class, expectedSubdomain == null); - assertThat(subdomain, equalTo(expectedSubdomain)); + public void testIgnoreMissing() throws Exception { + { + var processor = new RegisteredDomainProcessor(null, null, "domain", "", false); + IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), is("field [domain] not present as part of path [domain]")); + assertThat(document.getSource(), is(anEmptyMap())); + } + + { + var processor = new RegisteredDomainProcessor(null, null, "domain", "", true); + IngestDocument document = TestIngestDocument.withDefaultVersion(Collections.singletonMap("domain", null)); + processor.execute(document); + assertThat(document.getSource(), is(Collections.singletonMap("domain", null))); + } } } diff --git a/modules/ingest-geoip/qa/file-based-update/build.gradle b/modules/ingest-geoip/qa/file-based-update/build.gradle index 413a091726cdb..67eae701ce622 100644 --- a/modules/ingest-geoip/qa/file-based-update/build.gradle +++ b/modules/ingest-geoip/qa/file-based-update/build.gradle @@ -7,20 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'resource.reload.interval.high', '100ms' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' -} - -tasks.named("javaRestTest").configure { - systemProperty 'tests.security.manager', 'false' // Allows the test the add databases to config directory. - nonInputProperties.systemProperty 'tests.config.dir', testClusters.named("javaRestTest").map(c -> c.singleNode().getConfigDir()) -} - -tasks.named("forbiddenPatterns").configure { - exclude '**/*.mmdb' +dependencies { + clusterModules project(':modules:ingest-geoip') } diff --git a/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java index d75ce06b565ea..760aa218ff7c0 100644 --- a/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java +++ b/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java @@ -9,12 +9,12 @@ package org.elasticsearch.ingest.geoip; import org.elasticsearch.client.Request; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.PathUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.ObjectPath; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.nio.file.Files; @@ -29,6 +29,16 @@ import static org.hamcrest.Matchers.nullValue; public class UpdateDatabasesIT extends ESRestTestCase { + public static TemporaryFolder configDir = new TemporaryFolder(); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("ingest-geoip") + .withConfigDir(() -> configDir.getRoot().toPath()) + .setting("resource.reload.interval.high", "100ms") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(configDir).around(cluster); public void test() throws Exception { String body = """ @@ -51,7 +61,7 @@ public void test() throws Exception { assertThat(stats, nullValue()); } - Path configPath = PathUtils.get(System.getProperty("tests.config.dir")); + Path configPath = configDir.getRoot().toPath(); assertThat(Files.exists(configPath), is(true)); Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip"); Files.createDirectory(ingestGeoipDatabaseDir); @@ -82,9 +92,7 @@ public void test() throws Exception { } @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); } - } diff --git a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle index 1b3f7639a7be2..2fee38e78c25b 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle +++ b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle @@ -18,7 +18,7 @@ dependencies { javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) } -buildParams.bwcVersions.withWireCompatible(v -> v.before("8.0.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("8.16.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java index c6a9b20d453d4..d9d79da418600 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java +++ b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java @@ -12,34 +12,24 @@ import com.carrotsearch.randomizedtesting.annotations.Name; -import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.client.WarningsHandler; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.Nullable; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { @@ -48,29 +38,16 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas private static final GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); - // e.g. use ./gradlew -Dtests.jvm.argline="-Dgeoip_test_with_security=false" ":modules:ingest-geoip:qa:full-cluster-restart:check" - // to set this to false, if you so desire - private static final boolean useSecurity = Boolean.parseBoolean(System.getProperty("geoip_test_with_security", "true")); - private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .version(getOldClusterTestVersion()) .nodes(2) .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) - .setting("xpack.security.enabled", useSecurity ? "true" : "false") + .setting("xpack.security.enabled", "false") + // .setting("logger.org.elasticsearch.ingest.geoip", "TRACE") .feature(FeatureFlag.TIME_SERIES_MODE) .build(); - @Override - protected Settings restClientSettings() { - Settings settings = super.restClientSettings(); - if (useSecurity) { - String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); - settings = Settings.builder().put(settings).put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - return settings; - } - @ClassRule public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); @@ -83,150 +60,32 @@ protected ElasticsearchCluster getUpgradeCluster() { return cluster; } - public void testGeoIpSystemFeaturesMigration() throws Exception { - final List maybeSecurityIndex = useSecurity ? List.of(".security-7") : List.of(); - + @SuppressWarnings("unchecked") + public void testGeoIpDatabaseConfigurations() throws Exception { if (isRunningAgainstOldCluster()) { - Request enableDownloader = new Request("PUT", "/_cluster/settings"); - enableDownloader.setJsonEntity(""" - {"persistent": {"ingest.geoip.downloader.enabled": true}} - """); - assertOK(client().performRequest(enableDownloader)); - - Request putPipeline = new Request("PUT", "/_ingest/pipeline/geoip"); - putPipeline.setJsonEntity(""" + Request putConfiguration = new Request("PUT", "_ingest/ip_location/database/my-database-1"); + putConfiguration.setJsonEntity(""" { - "description": "Add geoip info", - "processors": [{ - "geoip": { - "field": "ip", - "target_field": "geo", - "database_file": "GeoLite2-Country.mmdb" - } - }] + "name": "GeoIP2-Domain", + "maxmind": { + "account_id": "1234567" + } } """); - assertOK(client().performRequest(putPipeline)); - - // wait for the geo databases to all be loaded - assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); - - // the geoip index should be created - assertBusy(() -> testCatIndices(List.of(".geoip_databases"), List.of())); - assertBusy(() -> testIndexGeoDoc()); - - // before the upgrade, Kibana should work - assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), List.of())); - } else { - // after the upgrade, but before the migration, Kibana should work - assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex)); - - // migrate the system features and give the cluster a moment to settle - Request migrateSystemFeatures = new Request("POST", "/_migration/system_features"); - assertOK(client().performRequest(migrateSystemFeatures)); - ensureHealth(request -> request.addParameter("wait_for_status", "yellow")); - - assertBusy(() -> testCatIndices(List.of(".geoip_databases-reindexed-for-9", "my-index-00001"), maybeSecurityIndex)); - assertBusy(() -> testIndexGeoDoc()); - - // after the migration, Kibana should work - if (useSecurity == false) { // BUT IT DOESN'T if security is enabled - assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex)); - } - - Request disableDownloader = new Request("PUT", "/_cluster/settings"); - disableDownloader.setJsonEntity(""" - {"persistent": {"ingest.geoip.downloader.enabled": false}} - """); - assertOK(client().performRequest(disableDownloader)); - - // the geoip index should be deleted - assertBusy(() -> testCatIndices(List.of("my-index-00001"), maybeSecurityIndex)); - - Request enableDownloader = new Request("PUT", "/_cluster/settings"); - enableDownloader.setJsonEntity(""" - {"persistent": {"ingest.geoip.downloader.enabled": true}} - """); - assertOK(client().performRequest(enableDownloader)); - - // wait for the geo databases to all be loaded - assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); - - // the geoip index should be recreated - assertBusy(() -> testCatIndices(List.of(".geoip_databases", "my-index-00001"), maybeSecurityIndex)); - assertBusy(() -> testIndexGeoDoc()); - } - } - - @SuppressWarnings("unchecked") - private void testDatabasesLoaded() throws IOException { - Request getTaskState = new Request("GET", "/_cluster/state"); - ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState)); - - List tasks = state.evaluate("metadata.persistent_tasks.tasks"); - // Short-circuit to avoid using steams if the list is empty - if (tasks.isEmpty()) { - fail(); - } - Map databases = (Map) tasks.stream().map(task -> { - try { - return ObjectPath.evaluate(task, "task.geoip-downloader.state.databases"); - } catch (IOException e) { - return null; - } - }).filter(Objects::nonNull).findFirst().orElse(null); - - assertNotNull(databases); - - for (String name : List.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb")) { - Object database = databases.get(name); - assertNotNull(database); - assertNotNull(ObjectPath.evaluate(database, "md5")); - } - } - - private void testCatIndices(List indexNames, @Nullable List additionalIndexNames) throws IOException { - Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all"); - String response = EntityUtils.toString(client().performRequest(catIndices).getEntity()); - List indices = List.of(response.trim().split("\\s+")); - - if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) { - indexNames = new ArrayList<>(indexNames); // recopy into a mutable list - indexNames.addAll(additionalIndexNames); - } - - assertThat(new HashSet<>(indices), is(new HashSet<>(indexNames))); - } - - private void testIndexGeoDoc() throws IOException { - Request putDoc = new Request("PUT", "/my-index-00001/_doc/my_id?pipeline=geoip"); - putDoc.setJsonEntity(""" - {"ip": "89.160.20.128"} - """); - assertOK(client().performRequest(putDoc)); - - Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id"); - ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc)); - assertNull(doc.evaluate("_source.tags")); - assertEquals("Sweden", doc.evaluate("_source.geo.country_name")); - } - - private void testGetStarAsKibana(List indexNames, @Nullable List additionalIndexNames) throws IOException { - Request getStar = new Request("GET", "*?expand_wildcards=all"); - getStar.setOptions( - RequestOptions.DEFAULT.toBuilder() - .addHeader("X-elastic-product-origin", "kibana") - .setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors - ); - Response response = client().performRequest(getStar); - assertOK(response); - - if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) { - indexNames = new ArrayList<>(indexNames); // recopy into a mutable list - indexNames.addAll(additionalIndexNames); + assertOK(client().performRequest(putConfiguration)); } - Map map = responseAsMap(response); - assertThat(map.keySet(), is(new HashSet<>(indexNames))); + assertBusy(() -> { + Request getConfiguration = new Request("GET", "_ingest/ip_location/database/my-database-1"); + Response response = assertOK(client().performRequest(getConfiguration)); + Map map = responseAsMap(response); + assertThat(map.keySet(), equalTo(Set.of("databases"))); + List> databases = (List>) map.get("databases"); + assertThat(databases, hasSize(1)); + Map database = databases.get(0); + assertThat(database.get("id"), is("my-database-1")); + assertThat(database.get("version"), is(1)); + assertThat(database.get("database"), equalTo(Map.of("name", "GeoIP2-Domain", "maxmind", Map.of("account_id", "1234567")))); + }, 30, TimeUnit.SECONDS); } } diff --git a/modules/ingest-geoip/qa/geoip-reindexed/build.gradle b/modules/ingest-geoip/qa/geoip-reindexed/build.gradle new file mode 100644 index 0000000000000..1b3f7639a7be2 --- /dev/null +++ b/modules/ingest-geoip/qa/geoip-reindexed/build.gradle @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.bwc-test' + + +dependencies { + javaRestTestImplementation project(':test:fixtures:geoip-fixture') + javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) +} + +buildParams.bwcVersions.withWireCompatible(v -> v.before("8.0.0")) { bwcVersion, baseName -> + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty("tests.old_cluster_version", bwcVersion) + } +} diff --git a/modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java b/modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java new file mode 100644 index 0000000000000..5ae2e449e8d3c --- /dev/null +++ b/modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java @@ -0,0 +1,291 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.ingest.geoip; + +import fixture.geoip.GeoIpHttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; +import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.is; + +public class GeoIpReindexedIT extends ParameterizedFullClusterRestartTestCase { + + private static final boolean useFixture = Boolean.getBoolean("geoip_use_service") == false; + + private static final GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); + + // e.g. use ./gradlew -Dtests.jvm.argline="-Dgeoip_test_with_security=false" ":modules:ingest-geoip:qa:full-cluster-restart:check" + // to set this to false, if you so desire + private static final boolean useSecurity = Boolean.parseBoolean(System.getProperty("geoip_test_with_security", "true")); + + private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) + .setting("xpack.security.enabled", useSecurity ? "true" : "false") + .feature(FeatureFlag.TIME_SERIES_MODE) + .build(); + + @Override + protected Settings restClientSettings() { + Settings settings = super.restClientSettings(); + if (useSecurity) { + String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); + settings = Settings.builder().put(settings).put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + return settings; + } + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + + public GeoIpReindexedIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + public void testGeoIpSystemFeaturesMigration() throws Exception { + final List maybeSecurityIndex = useSecurity ? List.of(".security-7") : List.of(); + + if (isRunningAgainstOldCluster()) { + Request enableDownloader = new Request("PUT", "/_cluster/settings"); + enableDownloader.setJsonEntity(""" + {"persistent": {"ingest.geoip.downloader.enabled": true}} + """); + assertOK(client().performRequest(enableDownloader)); + + Request putPipeline = new Request("PUT", "/_ingest/pipeline/geoip"); + putPipeline.setJsonEntity(""" + { + "description": "Add geoip info", + "processors": [{ + "geoip": { + "field": "ip", + "target_field": "geo", + "database_file": "GeoLite2-Country.mmdb" + } + }] + } + """); + assertOK(client().performRequest(putPipeline)); + + // wait for the geo databases to all be loaded + assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); + + // the geoip index should be created + assertBusy(() -> testCatIndices(List.of(".geoip_databases"), List.of())); + assertBusy(() -> testIndexGeoDoc()); + + // before the upgrade, Kibana should work + assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), List.of())); + + // as should a normal get * + assertBusy(() -> testGetStar(List.of("my-index-00001"), List.of())); + + // and getting data streams + assertBusy(() -> testGetDatastreams()); + } else { + // after the upgrade, but before the migration, Kibana should work + assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex)); + + // as should a normal get * + assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex)); + + // and getting data streams + assertBusy(() -> testGetDatastreams()); + + // migrate the system features and give the cluster a moment to settle + Request migrateSystemFeatures = new Request("POST", "/_migration/system_features"); + assertOK(client().performRequest(migrateSystemFeatures)); + ensureHealth(request -> request.addParameter("wait_for_status", "yellow")); + + assertBusy(() -> testCatIndices(List.of(".geoip_databases-reindexed-for-9", "my-index-00001"), maybeSecurityIndex)); + assertBusy(() -> testIndexGeoDoc()); + + // after the migration, Kibana should work + assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex)); + + // as should a normal get * + assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex)); + + // and getting data streams + assertBusy(() -> testGetDatastreams()); + + Request disableDownloader = new Request("PUT", "/_cluster/settings"); + disableDownloader.setJsonEntity(""" + {"persistent": {"ingest.geoip.downloader.enabled": false}} + """); + assertOK(client().performRequest(disableDownloader)); + + // the geoip index should be deleted + assertBusy(() -> testCatIndices(List.of("my-index-00001"), maybeSecurityIndex)); + + Request enableDownloader = new Request("PUT", "/_cluster/settings"); + enableDownloader.setJsonEntity(""" + {"persistent": {"ingest.geoip.downloader.enabled": true}} + """); + assertOK(client().performRequest(enableDownloader)); + + // wait for the geo databases to all be loaded + assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); + + // the geoip index should be recreated + assertBusy(() -> testCatIndices(List.of(".geoip_databases", "my-index-00001"), maybeSecurityIndex)); + assertBusy(() -> testIndexGeoDoc()); + } + } + + @SuppressWarnings("unchecked") + private void testDatabasesLoaded() throws IOException { + Request getTaskState = new Request("GET", "/_cluster/state"); + ObjectPath state = ObjectPath.createFromResponse(assertOK(client().performRequest(getTaskState))); + + List tasks = state.evaluate("metadata.persistent_tasks.tasks"); + // Short-circuit to avoid using steams if the list is empty + if (tasks.isEmpty()) { + fail(); + } + Map databases = (Map) tasks.stream().map(task -> { + try { + return ObjectPath.evaluate(task, "task.geoip-downloader.state.databases"); + } catch (IOException e) { + return null; + } + }).filter(Objects::nonNull).findFirst().orElse(null); + + assertNotNull(databases); + + for (String name : List.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb")) { + Object database = databases.get(name); + assertNotNull(database); + assertNotNull(ObjectPath.evaluate(database, "md5")); + } + } + + private void testCatIndices(List indexNames, @Nullable List additionalIndexNames) throws IOException { + Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all"); + // the cat APIs can sometimes 404, erroneously + // see https://github.com/elastic/elasticsearch/issues/104371 + setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND); + String response = EntityUtils.toString(assertOK(client().performRequest(catIndices)).getEntity()); + List indices = List.of(response.trim().split("\\s+")); + + if (additionalIndexNames != null) { + indexNames = CollectionUtils.concatLists(indexNames, additionalIndexNames); + } + + assertThat(new HashSet<>(indices), is(new HashSet<>(indexNames))); + } + + private void testIndexGeoDoc() throws IOException { + Request putDoc = new Request("PUT", "/my-index-00001/_doc/my_id?pipeline=geoip"); + putDoc.setJsonEntity(""" + {"ip": "89.160.20.128"} + """); + assertOK(client().performRequest(putDoc)); + + Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id"); + ObjectPath doc = ObjectPath.createFromResponse(assertOK(client().performRequest(getDoc))); + assertNull(doc.evaluate("_source.tags")); + assertEquals("Sweden", doc.evaluate("_source.geo.country_name")); + } + + private void testGetStar(List indexNames, @Nullable List additionalIndexNames) throws IOException { + Request getStar = new Request("GET", "*?expand_wildcards=all"); + getStar.setOptions( + RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors + ); + Response response = assertOK(client().performRequest(getStar)); + + if (additionalIndexNames != null) { + indexNames = CollectionUtils.concatLists(indexNames, additionalIndexNames); + } + + Map map = responseAsMap(response); + assertThat(map.keySet(), is(new HashSet<>(indexNames))); + } + + private void testGetStarAsKibana(List indexNames, @Nullable List additionalIndexNames) throws IOException { + Request getStar = new Request("GET", "*?expand_wildcards=all"); + getStar.setOptions( + RequestOptions.DEFAULT.toBuilder() + .addHeader("X-elastic-product-origin", "kibana") + .setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors + ); + Response response = assertOK(client().performRequest(getStar)); + + if (additionalIndexNames != null) { + indexNames = CollectionUtils.concatLists(indexNames, additionalIndexNames); + } + + Map map = responseAsMap(response); + assertThat(map.keySet(), is(new HashSet<>(indexNames))); + } + + private void testGetDatastreams() throws IOException { + final List> wildcardOptions = List.of( + List.of(), // the default for expand_wildcards (that is, the option is not specified) + List.of("all"), + List.of("none"), + List.of("hidden"), + List.of("open"), + List.of("closed"), + List.of("hidden", "open"), + List.of("hidden", "closed"), + List.of("open", "closed") + ); + for (List expandWildcards : wildcardOptions) { + final Request getStar = new Request( + "GET", + "_data_stream" + (expandWildcards.isEmpty() ? "" : ("?expand_wildcards=" + String.join(",", expandWildcards))) + ); + getStar.setOptions( + RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we only care about errors + ); + Response response = client().performRequest(getStar); + assertOK(response); + + // note: we don't actually care about the response, just that there was one and that it didn't error out on us + } + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java index 3a1efabb38eca..5b8461e5febd1 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -23,6 +24,8 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.MockSecureSettings; @@ -31,22 +34,27 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.ingest.EnterpriseGeoIpTask; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; +import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.After; import org.junit.ClassRule; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING; @@ -58,6 +66,8 @@ public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase { private static final String MAXMIND_DATABASE_TYPE = "GeoIP2-City"; private static final String IPINFO_DATABASE_TYPE = "asn"; + private static final String MAXMIND_CONFIGURATION = "test-1"; + private static final String IPINFO_CONFIGURATION = "test-2"; @ClassRule public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture( @@ -92,6 +102,10 @@ protected Collection> nodePlugins() { } @SuppressWarnings("unchecked") + @TestLogging( + reason = "understanding why ipinfo asn database sometimes is not loaded", + value = "org.elasticsearch.ingest.geoip.DatabaseNodeService:TRACE" + ) public void testEnterpriseDownloaderTask() throws Exception { /* * This test starts the enterprise geoip downloader task, and creates a database configuration. Then it creates an ingest @@ -140,6 +154,28 @@ public void testEnterpriseDownloaderTask() throws Exception { }); } + @After + public void cleanup() throws InterruptedException { + /* + * This method cleans up the database configurations that the test created. This allows the test to be run repeatedly. + */ + CountDownLatch latch = new CountDownLatch(1); + LatchedActionListener listener = new LatchedActionListener<>(ActionListener.noop(), latch); + SubscribableListener.newForked(l -> deleteDatabaseConfiguration(MAXMIND_CONFIGURATION, l)) + .andThen(l -> deleteDatabaseConfiguration(IPINFO_CONFIGURATION, l)) + .addListener(listener); + latch.await(10, TimeUnit.SECONDS); + } + + private void deleteDatabaseConfiguration(String configurationName, ActionListener listener) { + admin().cluster() + .execute( + DeleteDatabaseConfigurationAction.INSTANCE, + new DeleteDatabaseConfigurationAction.Request(TimeValue.MAX_VALUE, TimeValue.timeValueSeconds(10), configurationName), + listener + ); + } + private void startEnterpriseGeoIpDownloaderTask() { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); persistentTasksService.sendStartRequest( @@ -163,7 +199,7 @@ private void configureMaxmindDatabase(String databaseType) { new PutDatabaseConfigurationAction.Request( TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, - new DatabaseConfiguration("test-1", databaseType, new DatabaseConfiguration.Maxmind("test_account")) + new DatabaseConfiguration(MAXMIND_CONFIGURATION, databaseType, new DatabaseConfiguration.Maxmind("test_account")) ) ) .actionGet(); @@ -176,7 +212,7 @@ private void configureIpinfoDatabase(String databaseType) { new PutDatabaseConfigurationAction.Request( TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, - new DatabaseConfiguration("test-2", databaseType, new DatabaseConfiguration.Ipinfo()) + new DatabaseConfiguration(IPINFO_CONFIGURATION, databaseType, new DatabaseConfiguration.Ipinfo()) ) ) .actionGet(); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index 3e5e9b1e6a306..e37da5c257b25 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -678,7 +678,7 @@ private List getGeoIpTmpDirs() throws IOException { .map(DiscoveryNode::getId) .collect(Collectors.toSet()); // All nodes share the same geoip base dir in the shared tmp dir: - Path geoipBaseTmpDir = internalCluster().getDataNodeInstance(Environment.class).tmpFile().resolve("geoip-databases"); + Path geoipBaseTmpDir = internalCluster().getDataNodeInstance(Environment.class).tmpDir().resolve("geoip-databases"); assertThat(Files.exists(geoipBaseTmpDir), is(true)); final List geoipTmpDirs; try (Stream files = Files.list(geoipBaseTmpDir)) { @@ -690,7 +690,7 @@ private List getGeoIpTmpDirs() throws IOException { private void setupDatabasesInConfigDirectory() throws Exception { StreamSupport.stream(internalCluster().getInstances(Environment.class).spliterator(), false) - .map(Environment::configFile) + .map(Environment::configDir) .map(path -> path.resolve("ingest-geoip")) .distinct() .forEach(path -> { @@ -718,7 +718,7 @@ private void setupDatabasesInConfigDirectory() throws Exception { private void deleteDatabasesInConfigDirectory() throws Exception { StreamSupport.stream(internalCluster().getInstances(Environment.class).spliterator(), false) - .map(Environment::configFile) + .map(Environment::configDir) .map(path -> path.resolve("ingest-geoip")) .distinct() .forEach(path -> { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java index 3d2b54b04695f..289008236a852 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java @@ -42,7 +42,7 @@ final class ConfigDatabases implements Closeable { private final ConcurrentMap configDatabases; ConfigDatabases(Environment environment, GeoIpCache cache) { - this(environment.configFile().resolve("ingest-geoip"), cache); + this(environment.configDir().resolve("ingest-geoip"), cache); } ConfigDatabases(Path geoipConfigDir, GeoIpCache cache) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 940231b12c894..13958254b9020 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -114,7 +114,7 @@ public final class DatabaseNodeService implements IpDatabaseProvider { ClusterService clusterService ) { this( - environment.tmpFile(), + environment.tmpDir(), new OriginSettingClient(client, IngestService.INGEST_ORIGIN), cache, new ConfigDatabases(environment, cache), diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java index a50fe7dee9008..1452f1e1638d1 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -45,7 +45,7 @@ public final class IngestGeoIpMetadata implements Metadata.Custom { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "ingest_geoip_metadata", + TYPE, a -> new IngestGeoIpMetadata( ((List) a[0]).stream().collect(Collectors.toMap((m) -> m.database().id(), Function.identity())) ) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 3107f0bed55e8..2e0f3a11b3052 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -214,6 +214,11 @@ public List getRestHandlers( @Override public List getNamedXContent() { return List.of( + new NamedXContentRegistry.Entry( + Metadata.Custom.class, + new ParseField(IngestGeoIpMetadata.TYPE), + IngestGeoIpMetadata::fromXContent + ), new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskParams::fromXContent), new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskState::fromXContent), new NamedXContentRegistry.Entry( diff --git a/modules/ingest-geoip/src/main/plugin-metadata/entitlement-policy.yaml b/modules/ingest-geoip/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..a96c07a2d3b64 --- /dev/null +++ b/modules/ingest-geoip/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,11 @@ +org.elasticsearch.ingest.geoip: + - outbound_network + - files: + - relative_path: "ingest-geoip" + relative_to: config + mode: read +com.maxmind.db: + - files: + - relative_path: "ingest-geoip/" + relative_to: "config" + mode: "read_write" diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 6262c26cb752f..40aca7db2f5a0 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -38,7 +38,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin { @Override public Map getProcessors(Processor.Parameters parameters) { - Path userAgentConfigDirectory = parameters.env.configFile().resolve("ingest-user-agent"); + Path userAgentConfigDirectory = parameters.env.configDir().resolve("ingest-user-agent"); if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) { throw new IllegalStateException( diff --git a/modules/ingest-user-agent/src/main/plugin-metadata/entitlement-policy.yaml b/modules/ingest-user-agent/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..bee7f4570cc6d --- /dev/null +++ b/modules/ingest-user-agent/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,5 @@ +org.elasticsearch.ingest.useragent: + - files: + - relative_path: ingest-user-agent + relative_to: config + mode: read diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle index a7f30c4fbb5cd..ddab38e5eac4b 100644 --- a/modules/lang-expression/build.gradle +++ b/modules/lang-expression/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml index b05e6e3a7bf7c..29a083f06bbc8 100644 --- a/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,2 @@ -org.elasticsearch.script.expression: +org.apache.lucene.expressions: - create_class_loader diff --git a/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java index 87972cd6f2876..609e6c6ded429 100644 --- a/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java +++ b/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-expression").build(); + public LangExpressionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public LangExpressionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index bd1e084df5d38..28cd3a03bbd3b 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -6,9 +6,9 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java b/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java index e70e82bd6c14b..90cc018ba2643 100644 --- a/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java +++ b/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java @@ -11,7 +11,9 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; import java.io.IOException; @@ -19,6 +21,9 @@ public class SearchTemplateWithoutContentIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-mustache").build(); + public void testSearchTemplateMissingBody() throws IOException { ResponseException responseException = expectThrows( ResponseException.class, @@ -36,4 +41,9 @@ public void testMultiSearchTemplateMissingBody() throws IOException { assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode()); assertThat(responseException.getMessage(), containsString("request body or source parameter is required")); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index bc1cd30ad45bf..c327ba49e6d1c 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -423,7 +423,7 @@ public void testResultSizeLimit() throws IOException { ex.getCause().getCause(), allOf( instanceOf(SizeLimitingStringWriter.SizeLimitExceededException.class), - transformedMatch(Throwable::getMessage, endsWith("has exceeded the size limit [1024]")) + transformedMatch(Throwable::getMessage, endsWith("has size [1030] which exceeds the size limit [1024]")) ) ); } diff --git a/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java index 38c1c269b7599..dc3cb5253f28f 100644 --- a/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java +++ b/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-mustache").build(); + public LangMustacheClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public LangMustacheClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml index 5674d79b52a94..a7cf0a6bf9592 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml @@ -482,3 +482,118 @@ }] - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.root_cause.0.reason: "script score function must not produce negative scores, but got: [-9.0]"} + +--- + +"Script Sort + _score": + - do: + index: + index: test + id: "1" + body: { "test": "a", "num1": 1.0, "type" : "first" } + - do: + index: + index: test + id: "2" + body: { "test": "b", "num1": 2.0, "type" : "first" } + - do: + index: + index: test + id: "3" + body: { "test": "c", "num1": 3.0, "type" : "first" } + - do: + index: + index: test + id: "4" + body: { "test": "d", "num1": 4.0, "type" : "second" } + - do: + index: + index: test + id: "5" + body: { "test": "e", "num1": 5.0, "type" : "second" } + - do: + indices.refresh: {} + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + sort: [ + { + _script: { + script: { + lang: "painless", + source: "doc['num1'].value + _score" + }, + type: "number" + } + } + ] + + - match: { hits.total: 5 } + - match: { hits.hits.0.sort.0: 2.0 } + - match: { hits.hits.1.sort.0: 3.0 } + - match: { hits.hits.2.sort.0: 4.0 } + - match: { hits.hits.3.sort.0: 5.0 } + - match: { hits.hits.4.sort.0: 6.0 } + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + sort: [ + { + _script: { + script: { + lang: "painless", + source: "doc['test.keyword'].value + '-' + _score" + }, + type: "string" + } + } + ] + + - match: { hits.total: 5 } + - match: { hits.hits.0.sort.0: "a-1.0" } + - match: { hits.hits.1.sort.0: "b-1.0" } + - match: { hits.hits.2.sort.0: "c-1.0" } + - match: { hits.hits.3.sort.0: "d-1.0" } + - match: { hits.hits.4.sort.0: "e-1.0" } + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + aggs: + test: + terms: + field: type.keyword + aggs: + top_hits: + top_hits: + sort: [ + { + _script: { + script: { + lang: "painless", + source: "doc['test.keyword'].value + '-' + _score" + }, + type: "string" + } + }, + "_score" + ] + size: 1 + + - match: { hits.total: 5 } + - match: { aggregations.test.buckets.0.key: "first" } + - match: { aggregations.test.buckets.0.top_hits.hits.total: 3 } + - match: { aggregations.test.buckets.0.top_hits.hits.hits.0.sort.0: "a-1.0" } + - match: { aggregations.test.buckets.0.top_hits.hits.hits.0.sort.1: 1.0 } + - match: { aggregations.test.buckets.1.key: "second" } + - match: { aggregations.test.buckets.1.top_hits.hits.total: 2 } + - match: { aggregations.test.buckets.1.top_hits.hits.hits.0.sort.0: "d-1.0" } + - match: { aggregations.test.buckets.1.top_hits.hits.hits.0.sort.1: 1.0 } diff --git a/modules/legacy-geo/build.gradle b/modules/legacy-geo/build.gradle index d5e108eb3235b..2ad5ef66b6490 100644 --- a/modules/legacy-geo/build.gradle +++ b/modules/legacy-geo/build.gradle @@ -26,7 +26,7 @@ dependencies { testImplementation project(":test:framework") } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index b0634f0f1332f..a1e75668f8fbb 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; @@ -46,7 +47,6 @@ import org.elasticsearch.legacygeo.builders.ShapeBuilder; import org.elasticsearch.legacygeo.parsers.ShapeParser; import org.elasticsearch.legacygeo.query.LegacyGeoShapeQueryProcessor; -import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.locationtech.spatial4j.shape.Point; @@ -84,6 +84,7 @@ * "field" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) * * @deprecated use the field mapper in the spatial module + * TODO: Remove this class once we no longer need to supported reading 7.x indices that might have this field type */ @Deprecated public class LegacyGeoShapeFieldMapper extends AbstractShapeGeometryFieldMapper> { @@ -533,14 +534,9 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { } @Override - protected boolean isBoundsExtractionSupported() { - // Extracting bounds for geo shapes is not implemented yet. - return false; - } - - @Override - protected CoordinateEncoder coordinateEncoder() { - return CoordinateEncoder.GEO; + public BlockLoader blockLoader(BlockLoaderContext blContext) { + // Legacy geo-shapes do not support doc-values, we can only lead from source in ES|QL + return blockLoaderFromSource(blContext); } } diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index eee7b1b16c1db..055ad1d2a1ab6 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -21,9 +21,8 @@ restResources { } } -testClusters.configureEach { - // Modules who's integration is explicitly tested in integration tests - module ':modules:lang-mustache' +dependencies { + clusterModules project(':modules:lang-mustache') } tasks.named("yamlRestTestV7CompatTransform").configure({ task -> diff --git a/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java b/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java index bfaafd898082d..728c26f92feb9 100644 --- a/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java +++ b/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java @@ -12,10 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class RankEvalYamlIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-mustache").module("rank-eval").build(); + public RankEvalYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -24,4 +30,9 @@ public RankEvalYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 0f966ef413610..9b221902ef301 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -45,9 +45,6 @@ dependencies { clusterModules project(':modules:lang-painless') clusterModules project(':modules:parent-join') clusterModules project(":modules:rest-root") - - internalClusterTestImplementation project(':modules:lang-painless') - internalClusterTestImplementation project(':modules:lang-painless:spi') } restResources { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java index 914311e1190c1..ceba20570e7e5 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java @@ -106,7 +106,7 @@ protected List getSettingAsList(String key) throws Exception { return settings.getAsList(key); } }; - configuration = loader.load(environment.configFile()); + configuration = loader.load(environment.configDir()); reload(); final FileChangesListener listener = new FileChangesListener() { diff --git a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml index df557f9944253..d1e8d1aca74dd 100644 --- a/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,7 @@ ALL-UNNAMED: + - manage_threads - outbound_network + - files: + - relative_path: "" + relative_to: config + mode: read diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy index 2b6d821c4d1a8..016cc6365b6ee 100644 --- a/modules/reindex/src/main/plugin-metadata/plugin-security.policy +++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy @@ -10,9 +10,6 @@ grant { // reindex opens socket connections using the rest client permission java.net.SocketPermission "*", "connect"; - - // needed for Painless to generate runtime classes - permission java.lang.RuntimePermission "createClassLoader"; }; grant codeBase "${codebase.elasticsearch-rest-client}" { diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index 7894332bb6326..9f91516a9cf20 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -62,20 +62,20 @@ dependencies { api "com.github.stephenc.jcip:jcip-annotations:1.0-1" api "com.nimbusds:content-type:2.3" api "com.nimbusds:lang-tag:1.7" - api("com.nimbusds:nimbus-jose-jwt:9.37.3"){ + api("com.nimbusds:nimbus-jose-jwt:10.0.2"){ exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely } - api("com.nimbusds:oauth2-oidc-sdk:11.9.1"){ + api("com.nimbusds:oauth2-oidc-sdk:11.22.2"){ exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely } api "jakarta.activation:jakarta.activation-api:1.2.1" api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3" api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version - api "net.minidev:accessors-smart:2.5.0" - api "net.minidev:json-smart:2.5.0" + api "net.minidev:accessors-smart:2.5.2" + api "net.minidev:json-smart:2.5.2" api "org.codehaus.woodstox:stax2-api:4.2.2" - api "org.ow2.asm:asm:9.3" + api "org.ow2.asm:asm:9.7.1" runtimeOnly "com.google.code.gson:gson:2.11.0" runtimeOnly "org.cryptomator:siv-mode:1.5.2" @@ -189,11 +189,6 @@ tasks.named("thirdPartyAudit").configure { 'org.bouncycastle.cert.X509CertificateHolder', 'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder', 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', - 'org.bouncycastle.crypto.InvalidCipherTextException', - 'org.bouncycastle.crypto.engines.AESEngine', - 'org.bouncycastle.crypto.modes.GCMBlockCipher', - 'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider', - 'org.bouncycastle.jce.provider.BouncyCastleProvider', 'org.bouncycastle.openssl.PEMKeyPair', 'org.bouncycastle.openssl.PEMParser', 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter', diff --git a/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml index 74197fb3ed9ae..dc276909274ec 100644 --- a/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/repository-azure/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,18 @@ io.netty.common: - outbound_network + - manage_threads + - files: + - path: "/etc/os-release" + mode: "read" + - path: "/usr/lib/os-release" + mode: "read" + - path: "/proc/sys/net/core/somaxconn" + mode: read +com.azure.identity: + - outbound_network + - files: + - relative_path: "storage-azure/" #/config/storage-azure/azure-federated-token + relative_to: config + mode: read +reactor.core: + - manage_threads diff --git a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy index 8a7c623597376..3aeeb6bde3914 100644 --- a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -12,6 +12,8 @@ grant { permission java.net.SocketPermission "*", "connect"; // io.netty.util.concurrent.GlobalEventExecutor.startThread permission java.lang.RuntimePermission "setContextClassLoader"; + // io.netty.util.concurrent.GlobalEventExecutor.startThread + permission java.lang.RuntimePermission "getClassLoader"; // Used by jackson bean deserialization permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; diff --git a/modules/repository-gcs/src/main/config/log4j2.properties b/modules/repository-gcs/src/main/config/log4j2.properties new file mode 100644 index 0000000000000..3ba07198d91fd --- /dev/null +++ b/modules/repository-gcs/src/main/config/log4j2.properties @@ -0,0 +1,2 @@ +logger.entitlements_repository_gcs.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.repository-gcs.ALL-UNNAMED +logger.entitlements_repository_gcs.level = error diff --git a/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml index a1ff54f02d969..d91a8ae0daffb 100644 --- a/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ ALL-UNNAMED: - set_https_connection_properties # required by google-http-client + - outbound_network diff --git a/modules/repository-s3/src/main/config/log4j2.properties b/modules/repository-s3/src/main/config/log4j2.properties index 36a38cf9d13a0..903a5fde9d53c 100644 --- a/modules/repository-s3/src/main/config/log4j2.properties +++ b/modules/repository-s3/src/main/config/log4j2.properties @@ -12,3 +12,7 @@ logger.com_amazonaws_auth_profile_internal_BasicProfileConfigFileLoader.level = logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.name = com.amazonaws.services.s3.internal.UseArnRegionResolver logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.level = error + +logger.entitlements_repository_s3.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.repository-s3.ALL-UNNAMED +logger.entitlements_repository_s3.level = error + diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index da0f23c134b52..d5f7ef72ea4d0 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -358,7 +358,7 @@ static class CustomWebIdentityTokenCredentialsProvider implements AWSCredentials } // Make sure that a readable symlink to the token file exists in the plugin config directory // AWS_WEB_IDENTITY_TOKEN_FILE exists but we only use Web Identity Tokens if a corresponding symlink exists and is readable - Path webIdentityTokenFileSymlink = environment.configFile().resolve(WEB_IDENTITY_TOKEN_FILE_LOCATION); + Path webIdentityTokenFileSymlink = environment.configDir().resolve(WEB_IDENTITY_TOKEN_FILE_LOCATION); if (Files.exists(webIdentityTokenFileSymlink) == false) { LOGGER.warn( "Cannot use AWS Web Identity Tokens: AWS_WEB_IDENTITY_TOKEN_FILE is defined but no corresponding symlink exists " diff --git a/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml index df557f9944253..632c548c5b818 100644 --- a/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,12 @@ ALL-UNNAMED: + - manage_threads - outbound_network + - files: + - relative_path: "repository-s3/aws-web-identity-token-file" + relative_to: "config" + mode: "read" + # The security policy permission states this is "only for tests": org.elasticsearch.repositories.s3.S3RepositoryPlugin + # TODO: check this is actually needed, and if we can isolate it to a test-only policy + - write_system_properties: + properties: + - es.allow_insecure_settings diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java index 69fd0c0f5d6a7..2698eb718ded0 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java @@ -65,7 +65,7 @@ private static Environment getEnvironment() throws IOException { Files.createDirectory(configDirectory.resolve("repository-s3")); Files.writeString(configDirectory.resolve("repository-s3/aws-web-identity-token-file"), "YXdzLXdlYi1pZGVudGl0eS10b2tlbi1maWxl"); Environment environment = Mockito.mock(Environment.class); - Mockito.when(environment.configFile()).thenReturn(configDirectory); + Mockito.when(environment.configDir()).thenReturn(configDirectory); return environment; } @@ -212,7 +212,7 @@ public void testPickUpNewWebIdentityTokenWhenItsChanged() throws Exception { latch.countDown(); } }); - Files.writeString(environment.configFile().resolve("repository-s3/aws-web-identity-token-file"), newWebIdentityToken); + Files.writeString(environment.configDir().resolve("repository-s3/aws-web-identity-token-file"), newWebIdentityToken); safeAwait(latch); assertCredentials(awsCredentialsProvider.getCredentials()); diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index 51a223315644a..eca846f955bfd 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -158,7 +158,7 @@ private URL checkURL(URL urlToCheck) { if (normalizedUrl == null) { String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " + "path.repo setting or by {} setting: [{}] "; - logger.warn(logMessage, urlToCheck, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles()); + logger.warn(logMessage, urlToCheck, ALLOWED_URLS_SETTING.getKey(), environment.repoDirs()); String exceptionMessage = "file url [" + urlToCheck + "] doesn't match any of the locations specified by path.repo or " diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 4b64b9c56917d..145a4405d05f0 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -7,13 +7,14 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ + +import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin -import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.publish' @@ -48,6 +49,8 @@ dependencies { api "io.netty:netty-transport-native-unix-common:${versions.netty}" testImplementation project(':modules:rest-root') + + clusterModules project(':modules:rest-root') } restResources { @@ -77,28 +80,12 @@ TaskProvider pooledInternalClusterTest = tasks.register("pooledInternalClu setClasspath(internalTestSourceSet.getRuntimeClasspath()) } -TaskProvider pooledJavaRestTest = tasks.register("pooledJavaRestTest", RestIntegTestTask) { - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - SourceSet javaRestTestSourceSet = sourceSets.getByName(LegacyJavaRestTestPlugin.SOURCE_SET_NAME) - setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs()) - setClasspath(javaRestTestSourceSet.getRuntimeClasspath()) - - - testClusters.pooledJavaRestTest { - systemProperty 'es.use_unpooled_allocator', 'false' - } -} - tasks.named("internalClusterTest").configure { systemProperty 'es.insecure_network_trace_enabled', 'true' } -testClusters.configureEach { - module ':modules:rest-root' -} - tasks.named("check").configure { - dependsOn(pooledTest, pooledJavaRestTest, pooledInternalClusterTest) + dependsOn(pooledTest, pooledInternalClusterTest) } tasks.named("thirdPartyAudit").configure { diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/AbstractNetty4IT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/AbstractNetty4IT.java new file mode 100644 index 0000000000000..2d28d4c5e217b --- /dev/null +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/AbstractNetty4IT.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.rest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.MutableSystemPropertyProvider; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.List; + +public abstract class AbstractNetty4IT extends ESRestTestCase { + private static final MutableSystemPropertyProvider clusterSettings = new MutableSystemPropertyProvider(); + private final boolean usePooledAllocator; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("transport-netty4") + .module("rest-root") + .systemProperties(clusterSettings) + .build(); + + public AbstractNetty4IT(@Name("pooled") boolean pooledAllocator) { + this.usePooledAllocator = pooledAllocator; + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return List.of(new Object[] { true }, new Object[] { false }); + } + + @Before + public void maybeRestart() throws IOException { + // Restart the cluster to pick up the new setting if necessary + String current = clusterSettings.get(null).get("es.use_unpooled_allocator"); + if (current == null || current.equals(Boolean.toString(usePooledAllocator))) { + clusterSettings.get(null).put("es.use_unpooled_allocator", Boolean.toString(usePooledAllocator == false)); + cluster.restart(false); + closeClients(); + initClient(); + } + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java index ce30759de403d..d789eb8358872 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java @@ -9,6 +9,8 @@ package org.elasticsearch.rest; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -17,7 +19,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.http.HttpTransportSettings; -import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; import java.io.IOException; @@ -31,7 +32,11 @@ import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.matchesRegex; -public class Netty4BadRequestIT extends ESRestTestCase { +public class Netty4BadRequestIT extends AbstractNetty4IT { + + public Netty4BadRequestIT(@Name("pooled") boolean pooledAllocator) { + super(pooledAllocator); + } public void testBadRequest() throws IOException { final Response response = client().performRequest(new Request("GET", "/_nodes/settings")); diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java index 6eea5d09590f8..655e223855ba8 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java @@ -9,11 +9,12 @@ package org.elasticsearch.rest; +import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; -import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matcher; @@ -28,7 +29,12 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.nullValue; -public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase { +public class Netty4HeadBodyIsEmptyIT extends AbstractNetty4IT { + + public Netty4HeadBodyIsEmptyIT(@Name("pooled") boolean pooledAllocator) { + super(pooledAllocator); + } + public void testHeadRoot() throws IOException { headTestCase("/", emptyMap(), greaterThan(0)); headTestCase("/", singletonMap("pretty", ""), greaterThan(0)); diff --git a/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml b/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml index eb772a06423a3..685c0de3afe64 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/transport-netty4/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,6 +1,21 @@ io.netty.transport: - inbound_network - outbound_network + - manage_threads + # Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 + # the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! + # TODO: copied over from the security policy. Check if this is still valid + - write_system_properties: + properties: + - sun.nio.ch.bugLevel io.netty.common: - inbound_network - outbound_network + - manage_threads + - files: + - path: "/etc/os-release" + mode: "read" + - path: "/usr/lib/os-release" + mode: "read" + - path: "/proc/sys/net/core/somaxconn" + mode: read diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy index ed278af96d926..dbf8e728c1606 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -14,8 +14,9 @@ grant codeBase "${codebase.netty-common}" { // netty makes and accepts socket connections permission java.net.SocketPermission "*", "accept,connect"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; }; grant codeBase "${codebase.netty-transport}" { diff --git a/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java index 38b914a1413c8..7e9fc68b3f5d9 100644 --- a/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java +++ b/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java @@ -14,8 +14,10 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import org.apache.lucene.tests.util.TimeUnits; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; import java.io.IOException; @@ -23,6 +25,9 @@ @TimeoutSuite(millis = 30 * TimeUnits.MINUTE) public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("transport-netty4").build(); + public Netty4ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -37,4 +42,9 @@ public void test() throws IOException { assumeFalse("FIPS JVMs are configured to use the 'security4' transport rather than 'netty4'", inFipsJvm()); super.test(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/muted-tests.yml b/muted-tests.yml index fe4740168a9b3..86be6d8b8e8a4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -185,18 +185,9 @@ tests: - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT method: testTracingCrossCluster issue: https://github.com/elastic/elasticsearch/issues/112731 -- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT - method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only} - issue: https://github.com/elastic/elasticsearch/issues/115475 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests method: testProcessFileChanges issue: https://github.com/elastic/elasticsearch/issues/115280 -- class: org.elasticsearch.xpack.inference.InferenceCrudIT - method: testSupportedStream - issue: https://github.com/elastic/elasticsearch/issues/113430 -- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT - method: testGeoShapeGeoHash - issue: https://github.com/elastic/elasticsearch/issues/115664 - class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT issue: https://github.com/elastic/elasticsearch/issues/116126 - class: org.elasticsearch.upgrades.FullClusterRestartIT @@ -205,30 +196,12 @@ tests: - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT method: testSnapshotRestore {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/111799 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=search/380_sort_segments_on_timestamp/Test that index segments are NOT sorted on timestamp field when @timestamp field is dynamically added} - issue: https://github.com/elastic/elasticsearch/issues/116221 -- class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT - method: test {yaml=ingest/310_reroute_processor/Test remove then add reroute processor with and without lazy rollover} - issue: https://github.com/elastic/elasticsearch/issues/116158 -- class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT - method: test {yaml=ingest/310_reroute_processor/Test data stream with lazy rollover obtains pipeline from template} - issue: https://github.com/elastic/elasticsearch/issues/116157 -- class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT - method: testEnterpriseDownloaderTask - issue: https://github.com/elastic/elasticsearch/issues/115163 - class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT method: testDeprecatedSettingsReturnWarnings issue: https://github.com/elastic/elasticsearch/issues/108628 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT method: test {yaml=/10_apm/Test template reinstallation} issue: https://github.com/elastic/elasticsearch/issues/116445 -- class: org.elasticsearch.action.admin.HotThreadsIT - method: testHotThreadsDontFail - issue: https://github.com/elastic/elasticsearch/issues/115754 -- class: org.elasticsearch.action.search.PointInTimeIT - method: testPITTiebreak - issue: https://github.com/elastic/elasticsearch/issues/115810 - class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT method: test {p0=esql/61_enrich_ip/IP strings} issue: https://github.com/elastic/elasticsearch/issues/116529 @@ -241,15 +214,6 @@ tests: - class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT method: test {p0=esql/60_enrich/Enrich on keyword with fields} issue: https://github.com/elastic/elasticsearch/issues/116593 -- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT - method: testGeoShapeGeoTile - issue: https://github.com/elastic/elasticsearch/issues/115717 -- class: org.elasticsearch.search.StressSearchServiceReaperIT - method: testStressReaper - issue: https://github.com/elastic/elasticsearch/issues/115816 -- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT - method: testGeoShapeGeoHex - issue: https://github.com/elastic/elasticsearch/issues/115705 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNoDocValues SYNC} issue: https://github.com/elastic/elasticsearch/issues/116945 @@ -277,15 +241,21 @@ tests: - class: org.elasticsearch.xpack.inference.InferenceRestIT method: test {p0=inference/30_semantic_text_inference_bwc/Calculates embeddings using the default ELSER 2 endpoint} issue: https://github.com/elastic/elasticsearch/issues/117349 -- class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT - method: testSearchWithRandomDisconnects - issue: https://github.com/elastic/elasticsearch/issues/116175 -- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests - method: testMinimumVersionBetweenNewAndOldVersion - issue: https://github.com/elastic/elasticsearch/issues/117485 - class: org.elasticsearch.discovery.ClusterDisruptionIT method: testAckedIndexing issue: https://github.com/elastic/elasticsearch/issues/117024 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date.IN operator with null in list, finds match SYNC} + issue: https://github.com/elastic/elasticsearch/issues/121594 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date.IN operator with null in list, finds match ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/121594 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date.Implicit casting strings to dates for IN operator SYNC} + issue: https://github.com/elastic/elasticsearch/issues/121594 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date.Implicit casting strings to dates for IN operator ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/121594 # Examples: # @@ -338,21 +308,12 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117805 - class: org.elasticsearch.xpack.security.authc.ldap.UserAttributeGroupsResolverTests issue: https://github.com/elastic/elasticsearch/issues/116537 -- class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests - method: testFallbackIndicesOptions - issue: https://github.com/elastic/elasticsearch/issues/117937 - class: org.elasticsearch.repositories.s3.RepositoryS3EcsCredentialsRestIT method: testNonexistentBucketReadonlyFalse issue: https://github.com/elastic/elasticsearch/issues/118225 -- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT - method: testCancelRequestWhenFailingFetchingPages - issue: https://github.com/elastic/elasticsearch/issues/118213 - class: org.elasticsearch.reservedstate.service.RepositoriesFileSettingsIT method: testSettingsApplied issue: https://github.com/elastic/elasticsearch/issues/116694 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/20_reindex_status/Test Reindex With Existing Data Stream} - issue: https://github.com/elastic/elasticsearch/issues/118576 - class: org.elasticsearch.discovery.ec2.DiscoveryEc2AvailabilityZoneAttributeNoImdsIT method: testAvailabilityZoneAttribute issue: https://github.com/elastic/elasticsearch/issues/118564 @@ -368,18 +329,12 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set} issue: https://github.com/elastic/elasticsearch/issues/116777 -- class: org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT - method: testRelocation - issue: https://github.com/elastic/elasticsearch/issues/119280 - class: org.elasticsearch.xpack.esql.action.EsqlNodeFailureIT method: testFailureLoadingFields issue: https://github.com/elastic/elasticsearch/issues/118000 - class: org.elasticsearch.xpack.restart.FullClusterRestartIT method: testWatcherWithApiKey {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/119396 -- class: org.elasticsearch.search.profile.dfs.DfsProfilerIT - method: testProfileDfs - issue: https://github.com/elastic/elasticsearch/issues/119711 - class: org.elasticsearch.xpack.security.authc.ldap.ADLdapUserSearchSessionFactoryTests issue: https://github.com/elastic/elasticsearch/issues/119882 - class: org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapperTests @@ -388,36 +343,21 @@ tests: - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/119911 -- class: org.elasticsearch.xpack.ml.integration.ClassificationIT - method: testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsKeyword - issue: https://github.com/elastic/elasticsearch/issues/120071 - class: org.elasticsearch.oldrepos.OldRepositoryAccessIT method: testOldSourceOnlyRepoAccess issue: https://github.com/elastic/elasticsearch/issues/120080 -- class: org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT - method: testFeatureImportanceValues - issue: https://github.com/elastic/elasticsearch/issues/116564 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT method: test {date_nanos.Bucket Date nanos by 10 minutes} issue: https://github.com/elastic/elasticsearch/issues/120162 -- class: org.elasticsearch.xpack.inference.InferenceCrudIT - method: testGetServicesWithCompletionTaskType - issue: https://github.com/elastic/elasticsearch/issues/119959 - class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT method: testRandomDirectoryIOExceptions issue: https://github.com/elastic/elasticsearch/issues/118733 -- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT - method: test {p0=search.vectors/41_knn_search_bbq_hnsw/Vector rescoring has same scoring as exact search for kNN section} - issue: https://github.com/elastic/elasticsearch/issues/120441 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=logsdb/10_settings/missing hostname field} issue: https://github.com/elastic/elasticsearch/issues/120476 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests method: testBottomFieldSort issue: https://github.com/elastic/elasticsearch/issues/118214 -- class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests - method: testRetryPointInTime - issue: https://github.com/elastic/elasticsearch/issues/120442 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testMultipleInferencesTriggeringDownloadAndDeploy issue: https://github.com/elastic/elasticsearch/issues/117208 @@ -438,72 +378,101 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_start_stop/Test schedule_now on an already started transform} issue: https://github.com/elastic/elasticsearch/issues/120720 -- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncEnrichStopIT - method: testEnrichAfterStop - issue: https://github.com/elastic/elasticsearch/issues/120757 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT method: testStalledShardMigrationProperlyDetected issue: https://github.com/elastic/elasticsearch/issues/115697 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT method: testFileSettingsReprocessedOnRestartWithoutVersionChange issue: https://github.com/elastic/elasticsearch/issues/120964 -- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT - method: test {p0=data_stream/80_resolve_index_data_streams/Resolve index with hidden and closed indices} - issue: https://github.com/elastic/elasticsearch/issues/120965 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=snapshot/20_operator_privileges_disabled/Operator only settings can be set and restored by non-operator user when operator privileges is disabled} issue: https://github.com/elastic/elasticsearch/issues/120973 - class: org.elasticsearch.packaging.test.DockerTests issue: https://github.com/elastic/elasticsearch/issues/120978 -- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests - method: testActivateProfileForJWT - issue: https://github.com/elastic/elasticsearch/issues/120983 -- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests - method: testProfileIndexAutoCreation - issue: https://github.com/elastic/elasticsearch/issues/120987 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT method: testAuthenticateShouldNotFallThroughInCaseOfFailure issue: https://github.com/elastic/elasticsearch/issues/120902 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT method: testReservedStatePersistsOnRestart issue: https://github.com/elastic/elasticsearch/issues/120923 -- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT - method: test {p0=data_stream/140_data_stream_aliases/Create data stream alias} - issue: https://github.com/elastic/elasticsearch/issues/120920 -- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests - method: testSuggestProfilesWithHint - issue: https://github.com/elastic/elasticsearch/issues/121116 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=synonyms/110_synonyms_invalid/Reload index with an invalid synonym rule with lenient set to false} issue: https://github.com/elastic/elasticsearch/issues/121117 -- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT - method: test {yaml=cat.aliases/40_hidden/Test cat aliases output with a visible index with a hidden alias} - issue: https://github.com/elastic/elasticsearch/issues/121128 -- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT - method: test {p0=data_stream/140_data_stream_aliases/Create data stream aliases using wildcard expression} - issue: https://github.com/elastic/elasticsearch/issues/120890 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT issue: https://github.com/elastic/elasticsearch/issues/121143 -- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests - method: testActivateProfile - issue: https://github.com/elastic/elasticsearch/issues/121151 - class: org.elasticsearch.xpack.ml.integration.PyTorchModelIT issue: https://github.com/elastic/elasticsearch/issues/121165 -- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests - method: testClientSecretRotation - issue: https://github.com/elastic/elasticsearch/issues/120985 -- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT - method: test {p0=data_stream/140_data_stream_aliases/Create data stream alias with filter} - issue: https://github.com/elastic/elasticsearch/issues/121014 - class: org.elasticsearch.xpack.transform.integration.TransformAuditorIT method: testAuditorWritesAudits issue: https://github.com/elastic/elasticsearch/issues/121241 -- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT - method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/121115 -- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests - method: testGrantApiKeyForJWT - issue: https://github.com/elastic/elasticsearch/issues/121039 -- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests - method: testSuggestProfileWithData - issue: https://github.com/elastic/elasticsearch/issues/121258 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=transform/*} + issue: https://github.com/elastic/elasticsearch/issues/120816 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=ml/*} + issue: https://github.com/elastic/elasticsearch/issues/120816 +- class: org.elasticsearch.xpack.ml.packageloader.action.ModelLoaderUtilsTests + method: testSplitIntoRanges + issue: https://github.com/elastic/elasticsearch/issues/121799 +- class: org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests + method: testInvalidMaxAnalyzedOffset + issue: https://github.com/elastic/elasticsearch/issues/121361 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=data_stream/140_data_stream_aliases/Fix IndexNotFoundException error when handling remove alias action} + issue: https://github.com/elastic/elasticsearch/issues/121501 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/rest-api/security/invalidate-tokens/line_216} + issue: https://github.com/elastic/elasticsearch/issues/122229 +- class: org.elasticsearch.xpack.security.authc.esnative.ReservedRealmElasticAutoconfigIntegTests + method: testAutoconfigFailedPasswordPromotion + issue: https://github.com/elastic/elasticsearch/issues/122668 +- class: org.elasticsearch.xpack.core.common.notifications.AbstractAuditorTests + method: testRecreateTemplateWhenDeleted + issue: https://github.com/elastic/elasticsearch/issues/123232 +- class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT + method: test {yaml=analysis-common/40_token_filters/stemmer_override file access} + issue: https://github.com/elastic/elasticsearch/issues/121625 +- class: org.elasticsearch.smoketest.MlWithSecurityIT + method: test {yaml=ml/3rd_party_deployment/Test start deployment fails while model download in progress} + issue: https://github.com/elastic/elasticsearch/issues/120814 +- class: org.elasticsearch.gradle.internal.InternalBwcGitPluginFuncTest + method: current repository can be cloned + issue: https://github.com/elastic/elasticsearch/issues/123297 +- class: org.elasticsearch.gradle.internal.InternalBwcGitPluginFuncTest + method: can resolve checkout folder as project artifact + issue: https://github.com/elastic/elasticsearch/issues/119948 +- class: org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeToCharProcessorTests + issue: https://github.com/elastic/elasticsearch/issues/120575 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/cat/nodes/line_361} + issue: https://github.com/elastic/elasticsearch/issues/124103 +- class: org.elasticsearch.index.shard.StoreRecoveryTests + method: testAddIndices + issue: https://github.com/elastic/elasticsearch/issues/124104 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=snapshot/10_basic/Create a source only snapshot and then restore it} + issue: https://github.com/elastic/elasticsearch/issues/122755 +- class: org.elasticsearch.oldrepos.OldRepositoryAccessIT + method: testOldRepoAccess + issue: https://github.com/elastic/elasticsearch/issues/120148 +- class: org.elasticsearch.index.mapper.ShapeGeometryFieldMapperTests + method: testCartesianBoundsBlockLoader + issue: https://github.com/elastic/elasticsearch/issues/125129 +- class: org.elasticsearch.xpack.ilm.history.ILMHistoryItemTests + method: testTruncateLongError + issue: https://github.com/elastic/elasticsearch/issues/125216 +- class: org.elasticsearch.qa.verify_version_constants.VerifyVersionConstantsIT + method: testLuceneVersionConstant + issue: https://github.com/elastic/elasticsearch/issues/125638 +- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT + method: test {p0=/11_nodes/Additional disk information} + issue: https://github.com/elastic/elasticsearch/issues/125905 +- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT + method: test {p0=/10_info/Info} + issue: https://github.com/elastic/elasticsearch/issues/125904 +- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT + method: test {p0=/11_nodes/Test cat nodes output} + issue: https://github.com/elastic/elasticsearch/issues/125906 +- class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT + method: test {p0=/11_nodes/Test cat nodes output with full_id set} + issue: https://github.com/elastic/elasticsearch/issues/125903 diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 6fe503f0f5336..1bc9b8a3240b3 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -8,8 +8,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -28,7 +28,7 @@ dependencies { api "com.ibm.icu:icu4j:${versions.icu4j}" } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java index ae8ead523b7ea..fb83ac102392f 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java @@ -51,7 +51,7 @@ public IcuCollationTokenFilterFactory(IndexSettings indexSettings, Environment e if (rules != null) { Exception failureToResolve = null; try { - rules = Streams.copyToString(Files.newBufferedReader(environment.configFile().resolve(rules), Charset.forName("UTF-8"))); + rules = Streams.copyToString(Files.newBufferedReader(environment.configDir().resolve(rules), Charset.forName("UTF-8"))); } catch (IOException | SecurityException | InvalidPathException e) { failureToResolve = e; } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java index 62ab6d8792905..83b226f89dd5c 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java @@ -99,7 +99,7 @@ public RuleBasedBreakIterator getBreakIterator(int script) { // parse a single RBBi rule file private static BreakIterator parseRules(String filename, Environment env) throws IOException { - final Path path = env.configFile().resolve(filename); + final Path path = env.configDir().resolve(filename); String rules = Files.readAllLines(path).stream().filter((v) -> v.startsWith("#") == false).collect(Collectors.joining("\n")); return new RuleBasedBreakIterator(rules.toString()); diff --git a/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java b/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java index db2e70ab624ae..bf4078f1a4920 100644 --- a/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java +++ b/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class IcuClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-icu").build(); + public IcuClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public IcuClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandid public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index 82fa59e5773c3..b1bf38e6959ca 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.' diff --git a/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java b/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java index 75ae8a744c8fb..9de92bc30d70a 100644 --- a/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java +++ b/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class KuromojiClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-kuromoji").build(); + public KuromojiClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public KuromojiClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle index 6254a56f0657f..08dc1142e165e 100644 --- a/plugins/analysis-nori/build.gradle +++ b/plugins/analysis-nori/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.' diff --git a/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java b/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java index 52f3fa5944277..cf34b1ed2097a 100644 --- a/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java +++ b/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class NoriClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-nori").build(); + public NoriClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public NoriClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandi public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle index 018e2c0e52b8d..cc9038cd5b16c 100644 --- a/plugins/analysis-phonetic/build.gradle +++ b/plugins/analysis-phonetic/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.' diff --git a/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java b/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java index 66589ee523972..13b5ac92cb97d 100644 --- a/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java +++ b/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class PhoneticClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-phonetic").build(); + public PhoneticClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public PhoneticClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle index b4ac03935aab5..e7b568905350a 100644 --- a/plugins/analysis-smartcn/build.gradle +++ b/plugins/analysis-smartcn/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.' diff --git a/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java b/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java index e0eff356f8bb7..21760390256ef 100644 --- a/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java +++ b/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class SmartCNClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-smartcn").build(); + public SmartCNClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public SmartCNClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle index 0fb15ec7d36d1..c752cb422d611 100644 --- a/plugins/analysis-stempel/build.gradle +++ b/plugins/analysis-stempel/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.' diff --git a/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java b/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java index 6ef5d037577dd..716e9c824dbdb 100644 --- a/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java +++ b/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class StempelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-stempel").build(); + public StempelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public StempelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle index 2be48240a8875..a6414ae673e2c 100644 --- a/plugins/analysis-ukrainian/build.gradle +++ b/plugins/analysis-ukrainian/build.gradle @@ -6,8 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description = 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.' diff --git a/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java index 9a1d14a6ceee2..8f9ab6759e590 100644 --- a/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java +++ b/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("analysis-ukrainian").build(); + public UkrainianClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public UkrainianClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate test public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/build.gradle b/plugins/build.gradle index 32fd646ef0be8..90fc924d7f9d6 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -7,10 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -subprojects { - apply plugin: 'elasticsearch.internal-testclusters' -} - configurations { allPlugins } @@ -23,11 +19,9 @@ configure(subprojects.findAll { it.parent.path == project.path }) { esplugin { // for local ES plugins, the name of the plugin is the same as the directory name = project.name - - licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } - parent.artifacts.add('allPlugins', tasks.named('bundlePlugin')) } diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index 8608ee05c7712..88654f4a46907 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -9,7 +9,10 @@ import org.elasticsearch.gradle.internal.info.BuildParams * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' + +import org.elasticsearch.gradle.LoggedExec + +apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -50,14 +53,14 @@ restResources { } } -// needed to be consistent with ssl host checking -String host = InetAddress.getLoopbackAddress().getHostAddress() - -// location of keystore and files to generate it -File keystore = new File(project.buildDir, 'keystore/test-node.jks') - // generate the keystore TaskProvider createKey = tasks.register("createKey", LoggedExec) { + // needed to be consistent with ssl host checking + String host = InetAddress.getLoopbackAddress().getHostAddress() + + // location of keystore and files to generate it + File keystore = project.layout.buildDirectory.file('keystore/test-node.jks').get().asFile + doFirst { delete(keystore.parentFile) keystore.parentFile.mkdirs() @@ -75,10 +78,9 @@ TaskProvider createKey = tasks.register("createKey", LoggedExec) { '-keypass', 'keypass', '-storepass', 'keypass' } -//no unit tests -tasks.named("test").configure { enabled = false } + // add keystore to test classpath: it expects it there -tasks.named("processInternalClusterTestResources").configure { +tasks.named("processInternalClusterTestResources") { from createKey } diff --git a/plugins/discovery-azure-classic/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/discovery-azure-classic/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..df557f9944253 --- /dev/null +++ b/plugins/discovery-azure-classic/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +ALL-UNNAMED: + - outbound_network diff --git a/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java b/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java index 711d033ecfca2..09de0cfe04be3 100644 --- a/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java +++ b/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class DiscoveryAzureClassicClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("discovery-azure-classic").build(); + public DiscoveryAzureClassicClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public DiscoveryAzureClassicClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCa public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 28f0168b4184c..e1765fb256c8d 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -48,13 +48,12 @@ esplugin.bundleSpec.from('config/discovery-ec2') { } tasks.register("writeTestJavaPolicy") { + boolean inFips = buildParams.inFipsJvm + inputs.property("inFipsJvm", inFips) + final File javaPolicy = new File(layout.buildDirectory.asFile.get(), "tmp/java.policy") + outputs.file(javaPolicy) doLast { - final File tmp = file("${buildDir}/tmp") - if (tmp.exists() == false && tmp.mkdirs() == false) { - throw new GradleException("failed to create temporary directory [${tmp}]") - } - final File javaPolicy = file("${tmp}/java.policy") - if (buildParams.inFipsJvm) { + if (inFips) { javaPolicy.write( [ "grant {", @@ -97,9 +96,9 @@ tasks.withType(Test).configureEach { // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'` if (buildParams.inFipsJvm){ - nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy" + nonInputProperties.systemProperty 'java.security.policy', "=file://${layout.buildDirectory.asFile.get()}/tmp/java.policy" } else { - nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" + nonInputProperties.systemProperty 'java.security.policy', "file://${layout.buildDirectory.asFile.get()}/tmp/java.policy" } } diff --git a/plugins/discovery-ec2/config/discovery-ec2/log4j2.properties b/plugins/discovery-ec2/config/discovery-ec2/log4j2.properties index 36a38cf9d13a0..78e7c0aa473bb 100644 --- a/plugins/discovery-ec2/config/discovery-ec2/log4j2.properties +++ b/plugins/discovery-ec2/config/discovery-ec2/log4j2.properties @@ -12,3 +12,6 @@ logger.com_amazonaws_auth_profile_internal_BasicProfileConfigFileLoader.level = logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.name = com.amazonaws.services.s3.internal.UseArnRegionResolver logger.com_amazonaws_services_s3_internal_UseArnRegionResolver.level = error + +logger.entitlements_discovery_ec2.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.discovery-ec2.ALL-UNNAMED +logger.entitlements_discovery_ec2.level = error diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml index df557f9944253..394e5e38d9f59 100644 --- a/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml +++ b/plugins/discovery-ec2/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ ALL-UNNAMED: + - manage_threads - outbound_network diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index c6beaf3f332ca..4ac96920895db 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -44,9 +44,8 @@ tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' } -tasks.named("check").configure { - // also execute the QA tests when testing the plugin - dependsOn 'qa:gce:check' +esplugin.bundleSpec.from('config/discovery-gce') { + into 'config' } tasks.named("test").configure { diff --git a/plugins/discovery-gce/config/discovery-gce/log4j2.properties b/plugins/discovery-gce/config/discovery-gce/log4j2.properties new file mode 100644 index 0000000000000..cde35af628683 --- /dev/null +++ b/plugins/discovery-gce/config/discovery-gce/log4j2.properties @@ -0,0 +1,3 @@ +logger.entitlements_discovery_gce.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.discovery-gce.ALL-UNNAMED +logger.entitlements_discovery_gce.level = error + diff --git a/plugins/discovery-gce/qa/build.gradle b/plugins/discovery-gce/qa/build.gradle deleted file mode 100644 index f6a418ae4bd3c..0000000000000 --- a/plugins/discovery-gce/qa/build.gradle +++ /dev/null @@ -1 +0,0 @@ -group = "${group}.plugins.discovery-gce.qa" diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle deleted file mode 100644 index b55f17202d0cb..0000000000000 --- a/plugins/discovery-gce/qa/gce/build.gradle +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - - -import org.apache.tools.ant.filters.ReplaceTokens -import org.elasticsearch.gradle.internal.info.BuildParams -import org.elasticsearch.gradle.internal.test.AntFixture - -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.legacy-yaml-rest-test' - -final int gceNumberOfNodes = 3 - -dependencies { - yamlRestTestImplementation project(':plugins:discovery-gce') -} - -restResources { - restApi { - include '_common', 'cluster', 'nodes' - } -} - -/** A task to start the GCEFixture which emulates a GCE service **/ -def gceFixtureProvider = tasks.register("gceFixture", AntFixture) { - def runtimeClasspath = project.sourceSets.yamlRestTest.runtimeClasspath - dependsOn runtimeClasspath - env 'CLASSPATH', "${-> runtimeClasspath.asPath}" - executable = "${buildParams.runtimeJavaHome.get()}/bin/java" - args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, "${buildDir}/testclusters/yamlRestTest-1/config/unicast_hosts.txt" -} - -Map expansions = [ - 'expected_nodes': gceNumberOfNodes -] - -tasks.named("processYamlRestTestResources").configure { - inputs.properties(expansions) - filter("tokens" : expansions.collectEntries {k, v -> [k, v.toString()]} /* must be a map of strings */, ReplaceTokens.class) -} - -tasks.named("yamlRestTest").configure { - dependsOn gceFixtureProvider -} - -testClusters.matching { it.name == "yamlRestTest" }.configureEach { - numberOfNodes = gceNumberOfNodes - plugin ':plugins:discovery-gce' - // use gce fixture for Auth calls instead of http://metadata.google.internal - environment 'GCE_METADATA_HOST', { "http://${gceFixtureProvider.get().addressAndPort}" }, IGNORE_VALUE - // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`) - systemProperty 'es.allow_reroute_gce_settings', 'true' - - setting 'discovery.seed_providers', 'gce' - // use gce fixture for metadata server calls instead of http://metadata.google.internal - setting 'cloud.gce.host', { "http://${gceFixtureProvider.get().addressAndPort}" }, IGNORE_VALUE - // use gce fixture for API calls instead of https://www.googleapis.com - setting 'cloud.gce.root_url', { "http://${gceFixtureProvider.get().addressAndPort}" }, IGNORE_VALUE -} diff --git a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java deleted file mode 100644 index c645d0114331a..0000000000000 --- a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.cloud.gce; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; - -public class GCEDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - - public GCEDiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } -} diff --git a/plugins/discovery-gce/qa/gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml b/plugins/discovery-gce/qa/gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml deleted file mode 100644 index 62939470d686b..0000000000000 --- a/plugins/discovery-gce/qa/gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Integration tests for discovery-gce -setup: - - do: - cluster.health: - wait_for_status: green - wait_for_nodes: @expected_nodes@ - ---- -"All nodes are correctly discovered": - - - do: - nodes.info: - metric: [ transport ] - - - match: { _nodes.total: @expected_nodes@ } diff --git a/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml index a1ff54f02d969..d91a8ae0daffb 100644 --- a/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml +++ b/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,3 @@ ALL-UNNAMED: - set_https_connection_properties # required by google-http-client + - outbound_network diff --git a/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java b/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java index 718e8f511bdf7..8814e518f3062 100644 --- a/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java +++ b/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java @@ -12,11 +12,35 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +@SuppressForbidden(reason = "fixtures use java.io.File based APIs") public class DiscoveryGceClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + public static GCEFixture gceFixture = new GCEFixture(() -> temporaryFolder.getRoot().toPath().resolve("unicast_hosts.txt")); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .plugin("discovery-gce") + .nodes(3) + .node(0, n -> n.withConfigDir(() -> temporaryFolder.getRoot().toPath())) + .systemProperty("es.allow_reroute_gce_settings", "true") + .environment("GCE_METADATA_HOST", () -> gceFixture.getHostAndPort()) + .setting("discovery.seed_providers", "gce") + .setting("cloud.gce.host", () -> gceFixture.getAddress()) + .setting("cloud.gce.root_url", () -> gceFixture.getAddress()) + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(temporaryFolder).around(gceFixture).around(cluster); + public DiscoveryGceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +49,9 @@ public DiscoveryGceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java b/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/GCEFixture.java similarity index 79% rename from plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java rename to plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/GCEFixture.java index 7a5764e1fbd87..88851a98a1718 100644 --- a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java +++ b/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/GCEFixture.java @@ -6,30 +6,31 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.cloud.gce; +package org.elasticsearch.discovery.gce; import org.apache.http.client.methods.HttpGet; import org.elasticsearch.common.Strings; import org.elasticsearch.common.path.PathTrie; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestUtils; import org.elasticsearch.test.fixture.AbstractHttpFixture; import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import static java.nio.charset.StandardCharsets.UTF_8; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -43,24 +44,23 @@ public class GCEFixture extends AbstractHttpFixture { public static final String ZONE = "test-zone"; public static final String TOKEN = "1/fFAGRNJru1FTz70BzhT3Zg"; public static final String TOKEN_TYPE = "Bearer"; - + private final Supplier nodes; private final PathTrie handlers; - private final Path nodes; - - private GCEFixture(final String workingDir, final String nodesUriPath) { - super(workingDir); - this.nodes = toPath(Objects.requireNonNull(nodesUriPath)); + public GCEFixture(Supplier nodesUriPath) { this.handlers = defaultHandlers(); + this.nodes = nodesUriPath; } - public static void main(String[] args) throws Exception { - if (args == null || args.length != 2) { - throw new IllegalArgumentException("GCEFixture "); - } + @Override + protected void before() throws Throwable { + InetSocketAddress inetSocketAddress = resolveAddress("0.0.0.0", 0); + listen(inetSocketAddress, false); + } - final GCEFixture fixture = new GCEFixture(args[0], args[1]); - fixture.listen(); + @Override + protected void after() { + stop(); } private static String nonAuthPath(Request request) { @@ -128,30 +128,32 @@ private PathTrie defaultHandlers() { handlers.insert(authPath(HttpGet.METHOD_NAME, "/compute/v1/projects/{project}/zones/{zone}/instances"), request -> { final var items = new ArrayList>(); int count = 0; - for (String address : Files.readAllLines(nodes)) { - count++; - items.add( - Map.of( - "id", - Long.toString(9309873766405L + count), - "description", - "ES node" + count, - "name", - "test" + count, - "kind", - "compute#instance", - "machineType", - "n1-standard-1", - "networkInterfaces", - List.of( - Map.of("accessConfigs", Collections.emptyList(), "name", "nic0", "network", "default", "networkIP", address) - ), - "status", - "RUNNING", - "zone", - ZONE - ) - ); + if (Files.exists(nodes.get())) { + for (String address : Files.readAllLines(nodes.get())) { + count++; + items.add( + Map.of( + "id", + Long.toString(9309873766405L + count), + "description", + "ES node" + count, + "name", + "test" + count, + "kind", + "compute#instance", + "machineType", + "n1-standard-1", + "networkInterfaces", + List.of( + Map.of("accessConfigs", Collections.emptyList(), "name", "nic0", "network", "default", "networkIP", address) + ), + "status", + "RUNNING", + "zone", + ZONE + ) + ); + } } final String json = Strings.toString( @@ -215,8 +217,11 @@ private static Response newError(final RestStatus status, final String code, fin return new Response(status.getStatus(), JSON_CONTENT_TYPE, response.getBytes(UTF_8)); } - @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here") - private static Path toPath(final String dir) { - return Paths.get(dir); + private static InetSocketAddress resolveAddress(String address, int port) { + try { + return new InetSocketAddress(InetAddress.getByName(address), port); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } } } diff --git a/plugins/discovery-gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml b/plugins/discovery-gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml index a5379c2c68bed..3e1b9d2913648 100644 --- a/plugins/discovery-gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml +++ b/plugins/discovery-gce/src/yamlRestTest/resources/rest-api-spec/test/discovery_gce/10_basic.yml @@ -1,5 +1,9 @@ -# Integration tests for Discovery GCE components -# +setup: + - do: + cluster.health: + wait_for_status: green + wait_for_nodes: 3 +--- "Discovery GCE loaded": - skip: reason: "contains is a newly added assertion" @@ -14,3 +18,11 @@ nodes.info: {} - contains: { nodes.$master.plugins: { name: discovery-gce } } +--- +"All nodes are correctly discovered": + + - do: + nodes.info: + metric: [ transport ] + + - match: { _nodes.total: 3 } diff --git a/plugins/examples/custom-processor/build.gradle b/plugins/examples/custom-processor/build.gradle index 6c0281d899a4e..9358f4d4998ae 100644 --- a/plugins/examples/custom-processor/build.gradle +++ b/plugins/examples/custom-processor/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'custom-processor' description = 'An example plugin showing how to register a custom ingest processor' classname ='org.elasticsearch.example.customprocessor.ExampleProcessorPlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle index 2774bf6e75c78..556cddc8efae7 100644 --- a/plugins/examples/custom-settings/build.gradle +++ b/plugins/examples/custom-settings/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'custom-settings' description = 'An example plugin showing how to register custom settings' classname ='org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } testClusters.configureEach { diff --git a/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java b/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java index 243201f632c8f..a085973e82b0a 100644 --- a/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java +++ b/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java @@ -70,7 +70,7 @@ public class ExampleCustomSettingsConfig { public ExampleCustomSettingsConfig(final Environment environment) { // Elasticsearch config directory - final Path configDir = environment.configFile(); + final Path configDir = environment.configDir(); // Resolve the plugin's custom settings file final Path customSettingsYamlFile = configDir.resolve("custom-settings/custom.yml"); diff --git a/plugins/examples/custom-significance-heuristic/build.gradle b/plugins/examples/custom-significance-heuristic/build.gradle index f2f0cefa6d6f5..766bfa5014a56 100644 --- a/plugins/examples/custom-significance-heuristic/build.gradle +++ b/plugins/examples/custom-significance-heuristic/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'custom-significance-heuristic' description = 'An example plugin showing how to write and register a custom significance heuristic' classname ='org.elasticsearch.example.customsigheuristic.CustomSignificanceHeuristicPlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/custom-suggester/build.gradle b/plugins/examples/custom-suggester/build.gradle index a1cf345f5e819..bc5877f79a3d9 100644 --- a/plugins/examples/custom-suggester/build.gradle +++ b/plugins/examples/custom-suggester/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'custom-suggester' description = 'An example plugin showing how to write and register a custom suggester' classname ='org.elasticsearch.example.customsuggester.CustomSuggesterPlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } testClusters.configureEach { diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index e712035eabc7b..0c545a14c4d1f 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,12 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +<<<<<<< HEAD distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5 distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip +======= +distributionSha256Sum=fba8464465835e74f7270bbf43d6d8a8d7709ab0a43ce1aa3323f73e9aa0c612 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip +>>>>>>> e19b2264afb (Update Gradle wrapper to 8.13 (#122421)) networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index f501bd466ebe5..2b0df5c710a58 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -14,8 +14,8 @@ esplugin { description = 'An example whitelisting additional classes and methods in painless' classname ='org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin' extendedPlugins = ['lang-painless'] - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle index 023033349dd8c..4b39befb1dfdf 100644 --- a/plugins/examples/rescore/build.gradle +++ b/plugins/examples/rescore/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'example-rescore' description = 'An example plugin implementing rescore and verifying that plugins *can* implement rescore' classname ='org.elasticsearch.example.rescore.ExampleRescorePlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index 43590b166a545..872edf00617f5 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'rest-handler' description = 'An example plugin showing how to register a REST handler' classname ='org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle index 0fb1baaea2f03..159129872176b 100644 --- a/plugins/examples/script-expert-scoring/build.gradle +++ b/plugins/examples/script-expert-scoring/build.gradle @@ -13,8 +13,8 @@ esplugin { name = 'script-expert-scoring' description = 'An example script engine to use low level Lucene internals for expert scoring' classname ='org.elasticsearch.example.expertscript.ExpertScriptPlugin' - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle index faf32774a20ac..ea147cb730c5e 100644 --- a/plugins/examples/security-authorization-engine/build.gradle +++ b/plugins/examples/security-authorization-engine/build.gradle @@ -6,8 +6,8 @@ esplugin { description = 'An example spi extension plugin for security that implements an Authorization Engine' classname ='org.elasticsearch.example.AuthorizationEnginePlugin' extendedPlugins = ['x-pack-security'] - licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } dependencies { diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java index 523fdc51f6423..6bb39c3dff88b 100644 --- a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java +++ b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java @@ -11,13 +11,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse; import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse.Indices; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Subject; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; -import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesToCheck; -import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesCheckResult; import org.elasticsearch.xpack.core.security.authz.ResolvedIndices; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; @@ -33,12 +32,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.function.Supplier; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -85,10 +84,13 @@ public void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo au } @Override - public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, - AsyncSupplier indicesAsyncSupplier, - Map aliasOrIndexLookup, - ActionListener listener) { + public void authorizeIndexAction( + RequestInfo requestInfo, + AuthorizationInfo authorizationInfo, + AsyncSupplier indicesAsyncSupplier, + Metadata metadata, + ActionListener listener + ) { if (isSuperuser(requestInfo.getAuthentication().getEffectiveSubject().getUser())) { indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> { Map indexAccessControlMap = new HashMap<>(); diff --git a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java index dbf797e3d0899..040b32942ded4 100644 --- a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java +++ b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java @@ -11,9 +11,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; @@ -29,8 +28,6 @@ import org.elasticsearch.xpack.core.security.user.User; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import static org.hamcrest.Matchers.is; @@ -117,12 +114,13 @@ public void testAuthorizeClusterAction() { public void testAuthorizeIndexAction() { CustomAuthorizationEngine engine = new CustomAuthorizationEngine(); - Map indicesMap = new HashMap<>(); - indicesMap.put("index", new ConcreteIndex(IndexMetadata.builder("index") - .settings(Settings.builder().put("index.version.created", IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .build(), null)); + Metadata metadata = Metadata.builder().put(IndexMetadata.builder("index") + .settings(Settings.builder().put("index.version.created", IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), + false + ).build(); // authorized { RequestInfo requestInfo = @@ -136,7 +134,7 @@ public void testAuthorizeIndexAction() { PlainActionFuture resultFuture = new PlainActionFuture<>(); engine.authorizeIndexAction(requestInfo, authzInfo, listener -> listener.onResponse(new ResolvedIndices(Collections.singletonList("index"), Collections.emptyList())), - indicesMap, resultFuture); + metadata, resultFuture); IndexAuthorizationResult result = resultFuture.actionGet(); assertThat(result.isGranted(), is(true)); IndicesAccessControl indicesAccessControl = result.getIndicesAccessControl(); @@ -156,7 +154,7 @@ public void testAuthorizeIndexAction() { PlainActionFuture resultFuture = new PlainActionFuture<>(); engine.authorizeIndexAction(requestInfo, authzInfo, listener -> listener.onResponse(new ResolvedIndices(Collections.singletonList("index"), Collections.emptyList())), - indicesMap, resultFuture); + metadata, resultFuture); IndexAuthorizationResult result = resultFuture.actionGet(); assertThat(result.isGranted(), is(false)); IndicesAccessControl indicesAccessControl = result.getIndicesAccessControl(); diff --git a/plugins/examples/settings.gradle b/plugins/examples/settings.gradle index bbeb024957b75..6b20792e99f76 100644 --- a/plugins/examples/settings.gradle +++ b/plugins/examples/settings.gradle @@ -8,7 +8,7 @@ */ plugins { - id "com.gradle.develocity" version "3.18.1" + id "com.gradle.develocity" version "3.19.2" } // Include all subdirectories as example projects diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java index 8b4a9d6544b75..f636335701da5 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; @@ -52,7 +53,7 @@ protected List loadFieldValues( } @Override - protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) { + protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) { return new AnnotatedHighlighterAnalyzer(super.wrapAnalyzer(analyzer, maxAnalyzedOffset)); } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java index 61abd64e98a96..71ec82b1761c9 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText; import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper; import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset; import org.elasticsearch.lucene.search.uhighlight.Snippet; import org.elasticsearch.search.fetch.subphase.highlight.LimitTokenOffsetAnalyzer; import org.elasticsearch.test.ESTestCase; @@ -85,7 +86,7 @@ private void assertHighlightOneDoc( int noMatchSize, String[] expectedPassages, int maxAnalyzedOffset, - Integer queryMaxAnalyzedOffset + Integer queryMaxAnalyzedOffsetIn ) throws Exception { try (Directory dir = newDirectory()) { @@ -116,8 +117,9 @@ private void assertHighlightOneDoc( for (int i = 0; i < markedUpInputs.length; i++) { annotations[i] = AnnotatedText.parse(markedUpInputs[i]); } + QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(queryMaxAnalyzedOffsetIn, maxAnalyzedOffset); if (queryMaxAnalyzedOffset != null) { - wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset); + wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset.getNotNull()); } AnnotatedHighlighterAnalyzer hiliteAnalyzer = new AnnotatedHighlighterAnalyzer(wrapperAnalyzer); hiliteAnalyzer.setAnnotations(annotations); @@ -311,6 +313,19 @@ public void testExceedMaxAnalyzedOffset() throws Exception { e.getMessage() ); + // Same as before, but force using index maxOffset (20) as queryMaxOffset by passing -1. + assertHighlightOneDoc( + "text", + new String[] { "[Long Text exceeds](Long+Text+exceeds) MAX analyzed offset)" }, + query, + Locale.ROOT, + breakIterator, + 0, + new String[] { "Long Text [exceeds](_hit_term=exceeds) MAX analyzed offset)" }, + 20, + -1 + ); + assertHighlightOneDoc( "text", new String[] { "[Long Text Exceeds](Long+Text+Exceeds) MAX analyzed offset [Long Text Exceeds](Long+Text+Exceeds)" }, diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index b175f483fa724..94334d13f48f4 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -23,20 +23,35 @@ versions << [ 'hadoop': '3.4.1' ] +def patched = Attribute.of('patched', Boolean) + configurations { hdfsFixture2 hdfsFixture3 + compileClasspath { + attributes { + attribute(patched, true) + } + } + runtimeClasspath { + attributes { + attribute(patched, true) + } + } + testCompileClasspath { + attributes { + attribute(patched, true) + } + } + testRuntimeClasspath { + attributes { + attribute(patched, true) + } + } } dependencies { - api project(path: 'hadoop-client-api', configuration: 'default') - if (isEclipse) { - /* - * Eclipse can't pick up the shadow dependency so we point it at *something* - * so it can compile things. - */ - api project(path: 'hadoop-client-api') - } + api("org.apache.hadoop:hadoop-client-api:${versions.hadoop}") runtimeOnly "org.apache.hadoop:hadoop-client-runtime:${versions.hadoop}" implementation "org.apache.hadoop:hadoop-hdfs:${versions.hadoop}" api "com.google.protobuf:protobuf-java:${versions.protobuf}" @@ -70,6 +85,20 @@ dependencies { hdfsFixture2 project(path: ':test:fixtures:hdfs-fixture', configuration: 'shadowedHdfs2') hdfsFixture3 project(path: ':test:fixtures:hdfs-fixture', configuration: 'shadow') + + attributesSchema { + attribute(patched) + } + artifactTypes.getByName("jar") { + attributes.attribute(patched, false) + } + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.hdfs.HdfsClassPatcher) { + from.attribute(patched, false) + to.attribute(patched, true) + parameters { + matchingArtifacts = ["hadoop-client-api"] + } + } } restResources { @@ -191,6 +220,15 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor', 'org.apache.hadoop.thirdparty.protobuf.MessageSchema', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor', - 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor' + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm', + 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot', + 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer', + 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1', + 'org.apache.hadoop.io.nativeio.NativeIO', + 'org.apache.hadoop.service.launcher.InterruptEscalator', + 'org.apache.hadoop.service.launcher.IrqHandler', + 'org.apache.hadoop.util.SignalLogger$Handler' ) } diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle deleted file mode 100644 index 46b0d949cdee2..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/build.gradle +++ /dev/null @@ -1,54 +0,0 @@ -import org.gradle.api.file.ArchiveOperations - -apply plugin: 'elasticsearch.java' - -sourceSets { - patcher -} - -configurations { - thejar { - canBeResolved = true - } -} - -dependencies { - thejar("org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}") { - transitive = false - } - - patcherImplementation 'org.ow2.asm:asm:9.7.1' - patcherImplementation 'org.ow2.asm:asm-tree:9.7.1' -} - -def outputDir = layout.buildDirectory.dir("patched-classes") - -def patchTask = tasks.register("patchClasses", JavaExec) { - inputs.files(configurations.thejar).withPathSensitivity(PathSensitivity.RELATIVE) - inputs.files(sourceSets.patcher.output).withPathSensitivity(PathSensitivity.RELATIVE) - outputs.dir(outputDir) - classpath = sourceSets.patcher.runtimeClasspath - mainClass = 'org.elasticsearch.hdfs.patch.HdfsClassPatcher' - def thejar = configurations.thejar - doFirst { - args(thejar.singleFile, outputDir.get().asFile) - } -} - -interface InjectedArchiveOps { - @Inject ArchiveOperations getArchiveOperations() -} - -tasks.named('jar').configure { - dependsOn(configurations.thejar) - def injected = project.objects.newInstance(InjectedArchiveOps) - def thejar = configurations.thejar - from(patchTask) - from({ injected.getArchiveOperations().zipTree(thejar.singleFile) }) { - eachFile { - if (outputDir.get().file(it.relativePath.pathString).asFile.exists()) { - it.exclude() - } - } - } -} diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt deleted file mode 100644 index 62fc5816c996b..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt +++ /dev/null @@ -1,2 +0,0 @@ -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java deleted file mode 100644 index 8e06ec00faaff..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.hdfs.patch; - -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.ClassWriter; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.function.Function; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; - -import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; -import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; - -public class HdfsClassPatcher { - static final Map> patchers = Map.of( - "org/apache/hadoop/util/ShutdownHookManager.class", - ShutdownHookManagerPatcher::new, - "org/apache/hadoop/util/Shell.class", - ShellPatcher::new, - "org/apache/hadoop/security/UserGroupInformation.class", - SubjectGetSubjectPatcher::new, - "org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", - SubjectGetSubjectPatcher::new - ); - - public static void main(String[] args) throws Exception { - String jarPath = args[0]; - Path outputDir = Paths.get(args[1]); - - try (JarFile jarFile = new JarFile(new File(jarPath))) { - for (var patcher : patchers.entrySet()) { - JarEntry jarEntry = jarFile.getJarEntry(patcher.getKey()); - if (jarEntry == null) { - throw new IllegalArgumentException("path [" + patcher.getKey() + "] not found in [" + jarPath + "]"); - } - byte[] classToPatch = jarFile.getInputStream(jarEntry).readAllBytes(); - - ClassReader classReader = new ClassReader(classToPatch); - ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS); - classReader.accept(patcher.getValue().apply(classWriter), 0); - - Path outputFile = outputDir.resolve(patcher.getKey()); - Files.createDirectories(outputFile.getParent()); - Files.write(outputFile, classWriter.toByteArray()); - } - } - } -} diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java index ce6acd79a0bb9..e74d1a87959f2 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java @@ -81,7 +81,7 @@ class HdfsSecurityContext { * Expects keytab file to exist at {@code $CONFIG_DIR$/repository-hdfs/krb5.keytab} */ static Path locateKeytabFile(Environment environment) { - Path keytabPath = environment.configFile().resolve("repository-hdfs").resolve("krb5.keytab"); + Path keytabPath = environment.configDir().resolve("repository-hdfs").resolve("krb5.keytab"); try { if (Files.exists(keytabPath) == false) { throw new RuntimeException("Could not locate keytab at [" + keytabPath + "]."); diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml index b5020dc1b7468..21d5fed283531 100644 --- a/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml +++ b/plugins/repository-hdfs/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,5 +1,12 @@ ALL-UNNAMED: + - manage_threads + - inbound_network # required for kerberos principals which specify a host component - outbound_network + - load_native_libraries - write_system_properties: properties: - hadoop.home.dir + - files: + - relative_path: "repository-hdfs/" + relative_to: config + mode: read_write diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle index 727f9ed588673..f84d7757a33bf 100644 --- a/plugins/store-smb/build.gradle +++ b/plugins/store-smb/build.gradle @@ -6,7 +6,7 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { diff --git a/plugins/store-smb/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/store-smb/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..1022253171a11 --- /dev/null +++ b/plugins/store-smb/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,5 @@ +ALL-UNNAMED: + - files: + - relative_path: "indices/" + relative_to: data + mode: read_write diff --git a/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java b/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java index afb12c4c79967..894dadbfb7824 100644 --- a/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java +++ b/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java @@ -12,11 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class StoreSmbClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("store-smb").build(); + public StoreSmbClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +30,9 @@ public StoreSmbClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index 780f3994ce627..d912ccbe07454 100644 --- a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -101,8 +101,8 @@ private static MockTransportService startTransport( TransportSearchAction.TYPE.name(), EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchRequest::new, - (request, channel, task) -> channel.sendResponse( - new SearchResponse( + (request, channel, task) -> { + var searchResponse = new SearchResponse( SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), InternalAggregations.EMPTY, null, @@ -117,8 +117,13 @@ private static MockTransportService startTransport( 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY - ) - ) + ); + try { + channel.sendResponse(searchResponse); + } finally { + searchResponse.decRef(); + } + } ); newService.registerRequestHandler( ClusterStateAction.NAME, diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java index 2510208a496a9..34d15fa4aebba 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -42,6 +43,7 @@ public class ESPolicyUnitTests extends ESTestCase { @BeforeClass public static void setupPolicy() { + assumeTrue("test requires security manager to be supported", RuntimeVersionFeature.isSecurityManagerAvailable()); assumeTrue("test cannot run with security manager", System.getSecurityManager() == null); DEFAULT_POLICY = PolicyUtil.readPolicy(ESPolicy.class.getResource(POLICY_RESOURCE), TEST_CODEBASES); } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java index 08d7e3b45702b..bc8308f48e52d 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java @@ -103,23 +103,23 @@ public void testEnvironmentPaths() throws Exception { // check that all directories got permissions: // bin file: ro - assertExactPermissions(new FilePermission(environment.binFile().toString(), "read,readlink"), permissions); + assertExactPermissions(new FilePermission(environment.binDir().toString(), "read,readlink"), permissions); // lib file: ro - assertExactPermissions(new FilePermission(environment.libFile().toString(), "read,readlink"), permissions); + assertExactPermissions(new FilePermission(environment.libDir().toString(), "read,readlink"), permissions); // modules file: ro - assertExactPermissions(new FilePermission(environment.modulesFile().toString(), "read,readlink"), permissions); + assertExactPermissions(new FilePermission(environment.modulesDir().toString(), "read,readlink"), permissions); // config file: ro - assertExactPermissions(new FilePermission(environment.configFile().toString(), "read,readlink"), permissions); + assertExactPermissions(new FilePermission(environment.configDir().toString(), "read,readlink"), permissions); // plugins: ro - assertExactPermissions(new FilePermission(environment.pluginsFile().toString(), "read,readlink"), permissions); + assertExactPermissions(new FilePermission(environment.pluginsDir().toString(), "read,readlink"), permissions); // data paths: r/w - for (Path dataPath : environment.dataFiles()) { + for (Path dataPath : environment.dataDirs()) { assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions); } - assertExactPermissions(new FilePermission(environment.sharedDataFile().toString(), "read,readlink,write,delete"), permissions); + assertExactPermissions(new FilePermission(environment.sharedDataDir().toString(), "read,readlink,write,delete"), permissions); // logs: r/w - assertExactPermissions(new FilePermission(environment.logsFile().toString(), "read,readlink,write,delete"), permissions); + assertExactPermissions(new FilePermission(environment.logsDir().toString(), "read,readlink,write,delete"), permissions); // temp dir: r/w assertExactPermissions(new FilePermission(fakeTmpDir.toString(), "read,readlink,write,delete"), permissions); } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java index d8352669083d1..5a21cda02e7f3 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.plugins.PluginDescriptor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -40,6 +41,7 @@ public class PolicyUtilTests extends ESTestCase { @Before public void assumeSecurityManagerDisabled() { + assumeTrue("test requires security manager to be supported", RuntimeVersionFeature.isSecurityManagerAvailable()); assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null); } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java deleted file mode 100644 index f40c0707db7fd..0000000000000 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.plugins.cli; - -import org.elasticsearch.bootstrap.PluginPolicyInfo; -import org.elasticsearch.bootstrap.PolicyUtil; -import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.PropertyPermission; -import java.util.Set; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; - -/** Tests plugin manager security check */ -public class PluginSecurityTests extends ESTestCase { - - PluginPolicyInfo makeDummyPlugin(String policy, String... files) throws IOException { - Path plugin = createTempDir(); - Files.copy(this.getDataPath(policy), plugin.resolve(PluginDescriptor.ES_PLUGIN_POLICY)); - for (String file : files) { - Files.createFile(plugin.resolve(file)); - } - return PolicyUtil.getPluginPolicyInfo(plugin, createTempDir()); - } - - /** Test that we can parse the set of permissions correctly for a simple policy */ - public void testParsePermissions() throws Exception { - assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null); - Path scratch = createTempDir(); - PluginPolicyInfo info = makeDummyPlugin("simple-plugin-security.policy"); - Set actual = PluginSecurity.getPermissionDescriptions(info, scratch); - assertThat(actual, contains(PluginSecurity.formatPermission(new PropertyPermission("someProperty", "read")))); - } - - /** Test that we can parse the set of permissions correctly for a complex policy */ - public void testParseTwoPermissions() throws Exception { - assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null); - Path scratch = createTempDir(); - PluginPolicyInfo info = makeDummyPlugin("complex-plugin-security.policy"); - Set actual = PluginSecurity.getPermissionDescriptions(info, scratch); - assertThat( - actual, - containsInAnyOrder( - PluginSecurity.formatPermission(new RuntimePermission("getClassLoader")), - PluginSecurity.formatPermission(new RuntimePermission("setFactory")) - ) - ); - } - - /** Test that we can format some simple permissions properly */ - public void testFormatSimplePermission() throws Exception { - assertEquals( - "java.lang.RuntimePermission accessDeclaredMembers", - PluginSecurity.formatPermission(new RuntimePermission("accessDeclaredMembers")) - ); - } -} diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java index 7145f4b1d2d84..aa341c7128df9 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; @@ -44,16 +45,26 @@ public class FullClusterRestartDownsampleIT extends ParameterizedFullClusterRest protected static LocalClusterConfigProvider clusterConfig = c -> {}; - private static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .version(getOldClusterTestVersion()) - .nodes(2) - .setting("xpack.security.enabled", "false") - .setting("indices.lifecycle.poll_interval", "5s") - .apply(() -> clusterConfig) - .feature(FeatureFlag.TIME_SERIES_MODE) - .feature(FeatureFlag.FAILURE_STORE_ENABLED) - .build(); + private static ElasticsearchCluster cluster = buildCluster(); + + private static ElasticsearchCluster buildCluster() { + Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION); + var cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("indices.lifecycle.poll_interval", "5s") + .apply(() -> clusterConfig) + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.FAILURE_STORE_ENABLED); + + if (oldVersion.before(Version.fromString("8.18.0"))) { + cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper"); + cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService"); + } + return cluster.build(); + } @ClassRule public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 53574ad46716f..8c7d861572d37 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -15,6 +15,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.Build; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -33,6 +34,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.XContentTestUtils; @@ -107,18 +109,28 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas protected static LocalClusterConfigProvider clusterConfig = c -> {}; - private static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .version(getOldClusterTestVersion()) - .nodes(2) - .setting("path.repo", () -> repoDirectory.getRoot().getPath()) - .setting("xpack.security.enabled", "false") - // some tests rely on the translog not being flushed - .setting("indices.memory.shard_inactive_time", "60m") - .apply(() -> clusterConfig) - .feature(FeatureFlag.TIME_SERIES_MODE) - .feature(FeatureFlag.FAILURE_STORE_ENABLED) - .build(); + private static ElasticsearchCluster cluster = buildCluster(); + + private static ElasticsearchCluster buildCluster() { + Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION); + var cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("path.repo", () -> repoDirectory.getRoot().getPath()) + .setting("xpack.security.enabled", "false") + // some tests rely on the translog not being flushed + .setting("indices.memory.shard_inactive_time", "60m") + .apply(() -> clusterConfig) + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.FAILURE_STORE_ENABLED); + + if (oldVersion.before(Version.fromString("8.18.0"))) { + cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper"); + cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService"); + } + return cluster.build(); + } @ClassRule public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); @@ -633,13 +645,14 @@ public void testRollover() throws Exception { ) ); - // assertBusy to work around https://github.com/elastic/elasticsearch/issues/104371 - assertBusy( - () -> assertThat( - EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v&error_trace")).getEntity()), - containsString("testrollover-000002") - ) - ); + assertBusy(() -> { + Request catIndices = new Request("GET", "/_cat/indices?v&error_trace"); + // the cat APIs can sometimes 404, erroneously + // see https://github.com/elastic/elasticsearch/issues/104371 + setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND); + Response response = assertOK(client().performRequest(catIndices)); + assertThat(EntityUtils.toString(response.getEntity()), containsString("testrollover-000002")); + }); } Request countRequest = new Request("POST", "/" + index + "-*/_search"); diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java index 04a50939c8eb1..47560b4ef3701 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -33,17 +34,27 @@ public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { @ClassRule - public static final ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .version(getOldClusterTestVersion()) - .module("constant-keyword") - .module("data-streams") - .module("mapper-extras") - .module("x-pack-aggregate-metric") - .module("x-pack-stack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .build(); + public static final ElasticsearchCluster cluster = buildCluster(); + + private static ElasticsearchCluster buildCluster() { + Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION); + var cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial"); + + if (oldVersion.before(Version.fromString("8.18.0"))) { + cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper"); + cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService"); + } + return cluster.build(); + } public LogsIndexModeFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { super(upgradeStatus); diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java index ed5ac80604480..a268b71891a75 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java @@ -38,7 +38,7 @@ @TestCaseOrdering(FullClusterRestartTestOrdering.class) public abstract class ParameterizedFullClusterRestartTestCase extends ESRestTestCase { private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0"); - private static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version"); + protected static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version"); private static IndexVersion oldIndexVersion; private static boolean upgradeFailed = false; private static boolean upgraded = false; diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 0cd5609592f5b..50a1f66cc8ace 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -64,9 +64,6 @@ excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry // Excluded because they create dot-prefixed indices on older versions excludeList.add('indices.resolve_index/20_resolve_system_index/*') -// Can't work until auto-expand replicas is 0-1 for synonyms index -excludeList.add("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set") - def clusterPath = getPath() buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> @@ -85,6 +82,10 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'health.master_history.no_master_transitions_threshold', '10' } requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") + if (bwcVersion.before(Version.fromString("8.18.0"))) { + jvmArgs '-da:org.elasticsearch.index.mapper.DocumentMapper' + jvmArgs '-da:org.elasticsearch.index.mapper.MapperService' + } } tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) { diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index d47ba685b9834..3f0193e2e4c2d 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -81,8 +81,8 @@ public void testNoControllerSpawn() throws IOException { Environment environment = TestEnvironment.newEnvironment(settings); // This plugin will NOT have a controller daemon - Path plugin = environment.modulesFile().resolve("a_plugin"); - Files.createDirectories(environment.modulesFile()); + Path plugin = environment.modulesDir().resolve("a_plugin"); + Files.createDirectories(environment.modulesDir()); Files.createDirectories(plugin); PluginTestUtil.writePluginProperties( plugin, @@ -112,8 +112,8 @@ public void testNoControllerSpawn() throws IOException { * Two plugins - one with a controller daemon and one without. */ public void testControllerSpawn() throws Exception { - assertControllerSpawns(Environment::pluginsFile, false); - assertControllerSpawns(Environment::modulesFile, true); + assertControllerSpawns(Environment::pluginsDir, false); + assertControllerSpawns(Environment::modulesDir, true); } private void assertControllerSpawns(final Function pluginsDirFinder, boolean expectSpawn) throws Exception { @@ -132,8 +132,8 @@ private void assertControllerSpawns(final Function pluginsDir // this plugin will have a controller daemon Path plugin = pluginsDirFinder.apply(environment).resolve("test_plugin"); - Files.createDirectories(environment.modulesFile()); - Files.createDirectories(environment.pluginsFile()); + Files.createDirectories(environment.modulesDir()); + Files.createDirectories(environment.pluginsDir()); Files.createDirectories(plugin); PluginTestUtil.writePluginProperties( plugin, @@ -218,7 +218,7 @@ public void testControllerSpawnWithIncorrectDescriptor() throws IOException { Environment environment = TestEnvironment.newEnvironment(settings); - Path plugin = environment.modulesFile().resolve("test_plugin"); + Path plugin = environment.modulesDir().resolve("test_plugin"); Files.createDirectories(plugin); PluginTestUtil.writePluginProperties( plugin, @@ -251,10 +251,10 @@ public void testSpawnerHandlingOfDesktopServicesStoreFiles() throws IOException final Environment environment = TestEnvironment.newEnvironment(settings); - Files.createDirectories(environment.modulesFile()); - Files.createDirectories(environment.pluginsFile()); + Files.createDirectories(environment.modulesDir()); + Files.createDirectories(environment.pluginsDir()); - final Path desktopServicesStore = environment.modulesFile().resolve(".DS_Store"); + final Path desktopServicesStore = environment.modulesDir().resolve(".DS_Store"); Files.createFile(desktopServicesStore); final Spawner spawner = new Spawner(); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java index 0f0a599423092..e764132f60f33 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java @@ -141,12 +141,21 @@ private String getEnrollmentToken() throws Exception { ); } - final String tokenValue = result.stdout() + final List filteredResult = result.stdout() .lines() .filter(line -> line.startsWith("WARNING:") == false) - .findFirst() - .orElseThrow(() -> new AssertionError("Failed to find any non-warning output lines")); - enrollmentTokenHolder.set(tokenValue); + .filter(line -> line.matches("\\d{2}:\\d{2}:\\d{2}\\.\\d{3} \\[main\\].*") == false) + .toList(); + + if (filteredResult.size() > 1) { + throw new AssertionError( + "Result from elasticsearch-create-enrollment-token contains unexpected output. Output was: \n" + result.stdout() + ); + } else if (filteredResult.isEmpty()) { + throw new AssertionError("Failed to find any non-warning output lines. Output was: \n" + result.stdout()); + } + + enrollmentTokenHolder.set(filteredResult.get(0)); }, 30, TimeUnit.SECONDS); return enrollmentTokenHolder.get(); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index 808aec92fb35d..7bb45e6029409 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -187,7 +187,7 @@ public static void waitForElasticsearchToStart() { Thread.sleep(STARTUP_SLEEP_INTERVAL_MILLISECONDS); // Set COLUMNS so that `ps` doesn't truncate its output - psOutput = dockerShell.run("bash -c 'COLUMNS=2000 ps ax'").stdout(); + psOutput = dockerShell.run("bash -c 'COLUMNS=3000 ps ax'").stdout(); if (psOutput.contains("org.elasticsearch.bootstrap.Elasticsearch")) { isElasticsearchRunning = true; diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 2f717f201f248..5c9c9cdd74658 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -13,6 +13,7 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-test-artifact-base' apply plugin: 'elasticsearch.bwc-test' +apply plugin: 'elasticsearch.fwc-test' testArtifacts { registerTestArtifactFromSourceSet(sourceSets.javaRestTest) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java index 89137b3ff9814..bfb90a97328e4 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java @@ -15,6 +15,7 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; @@ -26,20 +27,30 @@ public abstract class AbstractRollingUpgradeTestCase extends ParameterizedRollin private static final TemporaryFolder repoDirectory = new TemporaryFolder(); - private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .version(getOldClusterTestVersion()) - .nodes(NODE_NUM) - .setting("path.repo", new Supplier<>() { - @Override - @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") - public String get() { - return repoDirectory.getRoot().getPath(); - } - }) - .setting("xpack.security.enabled", "false") - .feature(FeatureFlag.TIME_SERIES_MODE) - .build(); + private static final ElasticsearchCluster cluster = buildCluster(); + + private static ElasticsearchCluster buildCluster() { + Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION); + var cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(NODE_NUM) + .setting("path.repo", new Supplier<>() { + @Override + @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") + public String get() { + return repoDirectory.getRoot().getPath(); + } + }) + .setting("xpack.security.enabled", "false") + .feature(FeatureFlag.TIME_SERIES_MODE); + + if (oldVersion.before(Version.fromString("8.18.0"))) { + cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper"); + cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService"); + } + return cluster.build(); + } @ClassRule public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java index 9cc3fff5828c9..faa012adcc139 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java @@ -11,10 +11,11 @@ import com.carrotsearch.randomizedtesting.annotations.Name; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.XContentTestUtils; +import org.junit.BeforeClass; import java.util.Collections; import java.util.List; @@ -30,6 +31,11 @@ public FeatureUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { super(upgradedNodes); } + @BeforeClass + public static void ensureNotForwardCompatTest() { + assumeFalse("Only supported by bwc tests", Boolean.parseBoolean(System.getProperty("tests.fwc", "false"))); + } + public void testGetFeatureUpgradeStatus() throws Exception { final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " @@ -106,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception { // for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we // want to keep using versions here. We also assume that for non-semantic version migrations are not required. boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map( - v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION) + v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION) ).orElse(false); if (migrationNeeded) { assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED")); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java index a20981a119d8f..7ac5c14e861dc 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java @@ -36,7 +36,7 @@ public abstract class ParameterizedRollingUpgradeTestCase extends ESRestTestCase { protected static final int NODE_NUM = 3; - private static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version"); + protected static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version"); private static final Set upgradedNodes = new HashSet<>(); private static TestFeatureService oldClusterTestFeatureService = null; private static boolean upgradeFailed = false; @@ -103,8 +103,12 @@ public void upgradeNode() throws Exception { for (int n = 0; n < requestedUpgradedNodes; n++) { if (upgradedNodes.add(n)) { try { - logger.info("Upgrading node {} to version {}", n, Version.CURRENT); - getUpgradeCluster().upgradeNodeToVersion(n, Version.CURRENT); + Version upgradeVersion = System.getProperty("tests.new_cluster_version") == null + ? Version.CURRENT + : Version.fromString(System.getProperty("tests.new_cluster_version")); + + logger.info("Upgrading node {} to version {}", n, upgradeVersion); + getUpgradeCluster().upgradeNodeToVersion(n, upgradeVersion); } catch (Exception e) { upgradeFailed = true; throw e; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java index 5308fe85c1cab..07034618be4a6 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -456,7 +457,11 @@ public void testBBQVectorSearch() throws Exception { } """; // create index and index 10 random floating point vectors - createIndex(BBQ_INDEX_NAME, Settings.EMPTY, mapping); + createIndex( + BBQ_INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(), + mapping + ); index64DimVectors(BBQ_INDEX_NAME); // force merge the index client().performRequest(new Request("POST", "/" + BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1")); @@ -485,8 +490,8 @@ public void testBBQVectorSearch() throws Exception { Map response = search(searchRequest); assertThat(extractValue(response, "hits.total.value"), equalTo(7)); List> hits = extractValue(response, "hits.hits"); - assertThat(hits.get(0).get("_id"), equalTo("0")); - assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); + assertThat("hits: " + response, hits.get(0).get("_id"), equalTo("0")); + assertThat("hits: " + response, (double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); // search with knn searchRequest = new Request("POST", "/" + BBQ_INDEX_NAME + "/_search"); @@ -504,8 +509,12 @@ public void testBBQVectorSearch() throws Exception { response = search(searchRequest); assertThat(extractValue(response, "hits.total.value"), equalTo(2)); hits = extractValue(response, "hits.hits"); - assertThat(hits.get(0).get("_id"), equalTo("0")); - assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005)); + assertThat("expected: 0 received" + hits.get(0).get("_id") + " hits: " + response, hits.get(0).get("_id"), equalTo("0")); + assertThat( + "expected_near: 0.99 received" + hits.get(0).get("_score") + "hits: " + response, + (double) hits.get(0).get("_score"), + closeTo(0.9934857, 0.005) + ); } public void testFlatBBQVectorSearch() throws Exception { @@ -530,7 +539,11 @@ public void testFlatBBQVectorSearch() throws Exception { } """; // create index and index 10 random floating point vectors - createIndex(FLAT_BBQ_INDEX_NAME, Settings.EMPTY, mapping); + createIndex( + FLAT_BBQ_INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(), + mapping + ); index64DimVectors(FLAT_BBQ_INDEX_NAME); // force merge the index client().performRequest(new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1")); @@ -559,8 +572,8 @@ public void testFlatBBQVectorSearch() throws Exception { Map response = search(searchRequest); assertThat(extractValue(response, "hits.total.value"), equalTo(7)); List> hits = extractValue(response, "hits.hits"); - assertThat(hits.get(0).get("_id"), equalTo("0")); - assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); + assertThat("hits: " + response, hits.get(0).get("_id"), equalTo("0")); + assertThat("hits: " + response, (double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); // search with knn searchRequest = new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_search"); @@ -578,8 +591,12 @@ public void testFlatBBQVectorSearch() throws Exception { response = search(searchRequest); assertThat(extractValue(response, "hits.total.value"), equalTo(2)); hits = extractValue(response, "hits.hits"); - assertThat(hits.get(0).get("_id"), equalTo("0")); - assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005)); + assertThat("expected: 0 received" + hits.get(0).get("_id") + " hits: " + response, hits.get(0).get("_id"), equalTo("0")); + assertThat( + "expected_near: 0.99 received" + hits.get(0).get("_score") + "hits: " + response, + (double) hits.get(0).get("_score"), + closeTo(0.9934857, 0.005) + ); } private void index64DimVectors(String indexName) throws Exception { @@ -605,6 +622,7 @@ private void index64DimVectors(String indexName) throws Exception { assertOK(client().performRequest(indexRequest)); } // always refresh to ensure the data is visible + flush(indexName, true); refresh(indexName); } diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle index 8331cb7c8b1e8..50755e1c16432 100644 --- a/qa/smoke-test-http/build.gradle +++ b/qa/smoke-test-http/build.gradle @@ -7,15 +7,12 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-java-rest-test' -//apply plugin: 'elasticsearch.test-with-dependencies' +apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-java-rest-test' dependencies { - javaRestTestImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" - javaRestTestImplementation project(':modules:rest-root') -} - -testClusters.configureEach { - module ':modules:rest-root' - setting 'xpack.security.enabled', 'false' + internalClusterTestImplementation project(":test:framework") + internalClusterTestImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + internalClusterTestImplementation project(':modules:rest-root') + clusterModules project(":modules:rest-root") } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/BulkRestIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/BulkRestIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsNotSetIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/CorsNotSetIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsNotSetIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/CorsNotSetIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsRegexIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/CorsRegexIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsRegexIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/CorsRegexIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/HealthRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/HealthRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/HttpSmokeTestCase.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/HttpSmokeTestCase.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/HttpStatsIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/HttpStatsIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndexingPressureRestIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndexingPressureRestIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java similarity index 65% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java index 92fde6d7765cc..a90b04d54649c 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java +++ b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java @@ -12,23 +12,12 @@ import org.apache.http.client.methods.HttpGet; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction; import org.elasticsearch.client.Request; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; public class IndicesSegmentsRestCancellationIT extends BlockedSearcherRestCancellationTestCase { - @TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/88201", - value = "org.elasticsearch.http.BlockedSearcherRestCancellationTestCase:DEBUG" - + ",org.elasticsearch.transport.TransportService:TRACE" - ) public void testIndicesSegmentsRestCancellation() throws Exception { runTest(new Request(HttpGet.METHOD_NAME, "/_segments"), IndicesSegmentsAction.NAME); } - @TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/88201", - value = "org.elasticsearch.http.BlockedSearcherRestCancellationTestCase:DEBUG" - + ",org.elasticsearch.transport.TransportService:TRACE" - ) public void testCatSegmentsRestCancellation() throws Exception { runTest(new Request(HttpGet.METHOD_NAME, "/_cat/segments"), IndicesSegmentsAction.NAME); } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NoHandlerIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/NoHandlerIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NoHandlerIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/NoHandlerIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PointInTimeIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/PointInTimeIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PointInTimeIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/PointInTimeIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/RestActionCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/RestActionCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchErrorTraceIT.java similarity index 58% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchErrorTraceIT.java index 6f9ab8ccdfdec..962c4cb1f98eb 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java +++ b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchErrorTraceIT.java @@ -11,15 +11,21 @@ import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NByteArrayEntity; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Request; +import org.elasticsearch.search.ErrorTraceHelper; +import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.MockLog; import org.elasticsearch.transport.TransportMessageListener; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; +import org.junit.BeforeClass; import java.io.IOException; import java.nio.charset.Charset; @@ -31,6 +37,11 @@ public class SearchErrorTraceIT extends HttpSmokeTestCase { private AtomicBoolean hasStackTrace; + @BeforeClass + public static void setDebugLogLevel() { + Configurator.setLevel(SearchService.class, Level.DEBUG); + } + @Before private void setupMessageListener() { internalCluster().getDataNodeInstances(TransportService.class).forEach(ts -> { @@ -119,6 +130,63 @@ public void testSearchFailingQueryErrorTraceFalse() throws IOException { assertFalse(hasStackTrace.get()); } + public void testDataNodeDoesNotLogStackTraceWhenErrorTraceTrue() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + Request searchRequest = new Request("POST", "/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + + String errorTriggeringIndex = "test2"; + int numShards = getNumShards(errorTriggeringIndex).numPrimaries; + try (var mockLog = MockLog.capture(SearchService.class)) { + ErrorTraceHelper.addUnseenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); + + searchRequest.addParameter("error_trace", "true"); + getRestClient().performRequest(searchRequest); + mockLog.assertAllExpectationsMatched(); + } + } + + public void testDataNodeLogsStackTraceWhenErrorTraceFalseOrEmpty() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + Request searchRequest = new Request("POST", "/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + + String errorTriggeringIndex = "test2"; + int numShards = getNumShards(errorTriggeringIndex).numPrimaries; + try (var mockLog = MockLog.capture(SearchService.class)) { + ErrorTraceHelper.addSeenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); + + // error_trace defaults to false so we can test both cases with some randomization + if (randomBoolean()) { + searchRequest.addParameter("error_trace", "false"); + } + getRestClient().performRequest(searchRequest); + mockLog.assertAllExpectationsMatched(); + } + } + public void testMultiSearchFailingQueryErrorTraceDefault() throws IOException { hasStackTrace = new AtomicBoolean(); setupIndexWithDocs(); @@ -172,4 +240,59 @@ public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException { assertFalse(hasStackTrace.get()); } + + public void testDataNodeDoesNotLogStackTraceWhenErrorTraceTrueMultiSearch() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + XContentType contentType = XContentType.JSON; + MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add( + new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field"))) + ); + Request searchRequest = new Request("POST", "/_msearch"); + byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent()); + searchRequest.setEntity( + new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null)) + ); + + searchRequest.addParameter("error_trace", "true"); + + String errorTriggeringIndex = "test2"; + int numShards = getNumShards(errorTriggeringIndex).numPrimaries; + try (var mockLog = MockLog.capture(SearchService.class)) { + ErrorTraceHelper.addUnseenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); + + getRestClient().performRequest(searchRequest); + mockLog.assertAllExpectationsMatched(); + } + } + + public void testDataNodeLogsStackTraceWhenErrorTraceFalseOrEmptyMultiSearch() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + XContentType contentType = XContentType.JSON; + MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add( + new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field"))) + ); + Request searchRequest = new Request("POST", "/_msearch"); + byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent()); + searchRequest.setEntity( + new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null)) + ); + + // error_trace defaults to false so we can test both cases with some randomization + if (randomBoolean()) { + searchRequest.addParameter("error_trace", "false"); + } + + String errorTriggeringIndex = "test2"; + int numShards = getNumShards(errorTriggeringIndex).numPrimaries; + try (var mockLog = MockLog.capture(SearchService.class)) { + ErrorTraceHelper.addSeenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); + + getRestClient().performRequest(searchRequest); + mockLog.assertAllExpectationsMatched(); + } + } } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchRestCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchRestCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SystemIndexRestIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SystemIndexRestIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java similarity index 100% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java rename to qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AbstractHttpSmokeTestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AbstractHttpSmokeTestIT.java new file mode 100644 index 0000000000000..495fb0a6b5e33 --- /dev/null +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AbstractHttpSmokeTestIT.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +public abstract class AbstractHttpSmokeTestIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("rest-root").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java index ce897664bd8fc..9d7d2dd9d3c92 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -29,7 +28,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.hamcrest.Matchers.containsString; -public class AutoCreateIndexIT extends ESRestTestCase { +public class AutoCreateIndexIT extends AbstractHttpSmokeTestIT { /** * Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to false diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java index 981190f22ce28..d3e526e4192f0 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; @@ -23,7 +22,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; -public class HttpCompressionIT extends ESRestTestCase { +public class HttpCompressionIT extends AbstractHttpSmokeTestIT { private static final String GZIP_ENCODING = "gzip"; private static final String SAMPLE_DOCUMENT = """ diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java index 2ac07a3851ee5..19cb9f8e6fca6 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.test.rest.ESRestTestCase; import java.util.Arrays; import java.util.List; @@ -29,7 +28,7 @@ * methods on REST endpoints should respond with status code 405 for more * information. */ -public class RestHttpResponseHeadersIT extends ESRestTestCase { +public class RestHttpResponseHeadersIT extends AbstractHttpSmokeTestIT { /** * For an OPTIONS request to a valid REST endpoint, verify that a 200 HTTP diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle index fe8904da7c1c1..cc3b5c19ab4a9 100644 --- a/qa/smoke-test-ingest-disabled/build.gradle +++ b/qa/smoke-test-ingest-disabled/build.gradle @@ -6,13 +6,8 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' dependencies { - testImplementation project(':modules:ingest-common') -} - -testClusters.matching { it.name == "yamlRestTest" }.configureEach { - setting 'xpack.security.enabled', 'false' - setting 'node.roles', '[data,master,remote_cluster_client]' + clusterModules project(':modules:ingest-common') } diff --git a/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java index 4c050b776a16a..254e8bd8e24e6 100644 --- a/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java +++ b/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java @@ -12,11 +12,20 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class SmokeTestIngestDisabledClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("ingest-common") + .setting("xpack.security.enabled", "false") + .setting("node.roles", "[data,master,remote_cluster_client]") + .build(); + public SmokeTestIngestDisabledClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -26,4 +35,8 @@ public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle index 5482fa4f4df85..ca5bbebd6b3d4 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle +++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle @@ -7,22 +7,16 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' dependencies { - yamlRestTestImplementation project(':modules:lang-mustache') -} - -testClusters.configureEach { - setting 'xpack.security.enabled', 'false' - extraConfigFile 'ingest-geoip/GeoLite2-City.mmdb', file("src/yamlRestTest/resources/GeoLite2-City.mmdb") -} - -tasks.named("yamlRestTestTestingConventions").configure { - baseClass 'org.elasticsearch.ingest.AbstractScriptTestCase' - baseClass 'org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase' -} - -tasks.named("forbiddenPatterns").configure { - exclude '**/*.mmdb' + internalClusterTestImplementation project(':test:framework') + internalClusterTestImplementation project(':modules:lang-mustache') + clusterModules project(":modules:ingest-common") + clusterModules project(":modules:lang-mustache") + clusterModules project(":modules:reindex") + clusterModules project(":modules:data-streams") + clusterModules project(":modules:ingest-geoip") + clusterModules project(":modules:mapper-extras") } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java similarity index 100% rename from qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java rename to qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java similarity index 100% rename from qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java rename to qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java similarity index 100% rename from qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java rename to qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/scripts/master.painless b/qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/resources/scripts/master.painless similarity index 100% rename from qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/scripts/master.painless rename to qa/smoke-test-ingest-with-all-dependencies/src/internalClusterTest/resources/scripts/master.painless diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java index 9618381440d9d..5dc8640693790 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java @@ -12,11 +12,25 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class SmokeTestIngestWithAllDepsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("ingest-common") + .module("lang-mustache") + .module("reindex") + .module("data-streams") + .module("ingest-geoip") + .module("mapper-extras") + .configFile("ingest-geoip/GeoLite2-City.mmdb", Resource.fromClasspath("GeoLite2-City.mmdb")) + .build(); + public SmokeTestIngestWithAllDepsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -26,4 +40,8 @@ public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index d4843fb152888..d602607bf7515 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -860,9 +860,9 @@ setup: - do: allowed_warnings: - - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + - "index template [foo_index_template] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [foo_index_template] will take precedence during new index creation" indices.put_index_template: - name: test-composable-1 + name: foo_index_template body: index_patterns: - foo* diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index c707c2b5e8c80..42151668922fe 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -9,26 +9,32 @@ import org.apache.tools.ant.filters.ReplaceTokens -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' ext.pluginPaths = [] +ext.pluginNames = [] project(':plugins').getChildProjects().each { pluginName, pluginProject -> pluginPaths << pluginProject.path + pluginNames << pluginName } -testClusters.matching { it.name == "yamlRestTest" }.configureEach { +dependencies { + clusterModules project(":modules:lang-painless") pluginPaths.each { pluginPath -> - plugin pluginPath + clusterPlugins(project(pluginPath)) } - setting 'xpack.security.enabled', 'false' } ext.expansions = [ 'expected.plugins.count': pluginPaths.size() ] -tasks.named("processYamlRestTestResources").configure { +tasks.named("processYamlRestTestResources") { assert pluginPaths.size() > 0 inputs.properties(expansions) filter("tokens" : expansions.collectEntries {k, v -> [k, v.toString()]} /* must be a map of strings */, ReplaceTokens.class) } + +tasks.named("yamlRestTest") { + systemProperty('tests.plugin.names', pluginNames.join(',')) +} diff --git a/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java index 586f0bfcbc712..fa2972a49ac02 100644 --- a/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java +++ b/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java @@ -12,11 +12,20 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().apply(c -> { + for (String plugin : System.getProperty("tests.plugin.names").split(",")) { + c.plugin(plugin); + } + }).module("lang-painless").setting("xpack.security.enabled", "false").build(); + public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -25,4 +34,9 @@ public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandida public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/qa/system-indices/build.gradle b/qa/system-indices/build.gradle index c619d4f02e527..4d21ad0505672 100644 --- a/qa/system-indices/build.gradle +++ b/qa/system-indices/build.gradle @@ -8,19 +8,18 @@ */ apply plugin: 'elasticsearch.base-internal-es-plugin' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name = 'system-indices-qa' description = 'Plugin for performing QA of system indices' classname ='org.elasticsearch.system.indices.SystemIndicesQA' - licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.autoconfiguration.enabled', 'false' - user username: 'rest_user', password: 'rest-user-password' +dependencies { + clusterModules project(':modules:analysis-common') + clusterModules project(':modules:ingest-common') + clusterModules project(':x-pack:plugin:migrate') } diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java new file mode 100644 index 0000000000000..2db5ea8aa838f --- /dev/null +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.system.indices; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +public abstract class AbstractSystemIndicesIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .plugin("system-indices-qa") + .module("analysis-common") + .module("ingest-common") + .module("x-pack-migrate") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java index 25cfbd7949611..62bc23ee4ad87 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java @@ -12,12 +12,8 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.XContentTestUtils; -import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.After; import java.util.Collections; @@ -29,20 +25,13 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -public class FeatureUpgradeApiIT extends ESRestTestCase { - - static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("rest-user-password".toCharArray())); +public class FeatureUpgradeApiIT extends AbstractSystemIndicesIT { @After public void resetFeatures() throws Exception { client().performRequest(new Request("POST", "/_features/_reset")); } - @Override - protected Settings restClientSettings() { - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); - } - public void testCreatingSystemIndex() throws Exception { var request = new Request("PUT", "/_net_new_sys_index/_create"); request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java index d2c48d1e9b609..1f497322647d4 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java @@ -15,24 +15,13 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.After; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; -public class NetNewSystemIndicesIT extends ESRestTestCase { - - static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("rest-user-password".toCharArray())); - - @Override - protected Settings restClientSettings() { - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); - } +public class NetNewSystemIndicesIT extends AbstractSystemIndicesIT { public void testCreatingSystemIndex() throws Exception { ResponseException e = expectThrows( diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java index 6109a1cf9dc31..1a724fcab2b07 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java @@ -12,10 +12,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.After; import java.io.IOException; @@ -26,19 +22,13 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SystemAliasIT extends ESRestTestCase { - static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("rest-user-password".toCharArray())); +public class SystemAliasIT extends AbstractSystemIndicesIT { @After public void resetFeatures() throws Exception { client().performRequest(new Request("POST", "/_features/_reset")); } - @Override - protected Settings restClientSettings() { - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); - } - public void testCreatingSystemIndexWithAlias() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle index ee29da53dc51b..e7d9ac3e1ecc5 100644 --- a/qa/verify-version-constants/build.gradle +++ b/qa/verify-version-constants/build.gradle @@ -7,58 +7,14 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.bwc-test' -dependencies { - testImplementation project(':modules:rest-root') -} - buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> - def baseCluster = testClusters.register(baseName) { - version = bwcVersion.toString() - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - } - - tasks.register("${baseName}#integTest", StandaloneRestIntegTestTask) { - useCluster baseCluster - nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.clustername', "${->baseCluster.get().getName()}") - } - - tasks.register(bwcTaskName(bwcVersion)) { - dependsOn "${baseName}#integTest" + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty 'tests.cluster_version', bwcVersion } } - -tasks.register("verifyDocsLuceneVersion") { - doFirst { - File docsVersionsFile = rootProject.file('docs/Versions.asciidoc') - List versionLines = docsVersionsFile.readLines('UTF-8') - String docsLuceneVersion = null - for (String line : versionLines) { - if (line.startsWith(':lucene_version:')) { - docsLuceneVersion = line.split()[1] - } - } - if (docsLuceneVersion == null) { - throw new GradleException('Could not find lucene version in docs version file') - } - String expectedLuceneVersion = VersionProperties.lucene - // remove potential -snapshot-{gitrev} suffix - expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/ - if (docsLuceneVersion != expectedLuceneVersion) { - throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]") - } - } -} - -tasks.named("check").configure { - dependsOn "verifyDocsLuceneVersion" -} diff --git a/qa/verify-version-constants/src/javaRestTest/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/javaRestTest/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java new file mode 100644 index 0000000000000..941d34673c2b2 --- /dev/null +++ b/qa/verify-version-constants/src/javaRestTest/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.qa.verify_version_constants; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.hamcrest.Matchers; +import org.junit.ClassRule; + +import java.io.IOException; +import java.text.ParseException; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +public class VerifyVersionConstantsIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(System.getProperty("tests.cluster_version")) + .setting("xpack.security.enabled", "false") + .build(); + + public void testLuceneVersionConstant() throws IOException, ParseException { + Response response = client().performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + + String luceneVersionString = objectPath.evaluate("version.lucene_version").toString(); + org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString); + + IndexVersion indexVersion = getIndexVersion(); + assertThat(indexVersion.luceneVersion(), equalTo(luceneVersion)); + } + + private IndexVersion getIndexVersion() throws IOException { + IndexVersion indexVersion = null; + + Request request = new Request("GET", "_nodes"); + request.addParameter("filter_path", "nodes.*.index_version,nodes.*.name"); + Response response = client().performRequest(request); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + Map nodeMap = objectPath.evaluate("nodes"); + for (String id : nodeMap.keySet()) { + Number ix = objectPath.evaluate("nodes." + id + ".index_version"); + IndexVersion version; + if (ix != null) { + version = IndexVersion.fromId(ix.intValue()); + } else { + // it doesn't have index version (pre 8.11) - just infer it from the release version + version = parseLegacyVersion(System.getProperty("tests.cluster_version")).map(x -> IndexVersion.fromId(x.id())) + .orElse(IndexVersions.MINIMUM_COMPATIBLE); + } + + if (indexVersion == null) { + indexVersion = version; + } else { + String name = objectPath.evaluate("nodes." + id + ".name"); + assertThat("Node " + name + " has a different index version to other nodes", version, Matchers.equalTo(indexVersion)); + } + } + + assertThat("Index version could not be read", indexVersion, notNullValue()); + return indexVersion; + } + + @Override + public boolean preserveClusterUponCompletion() { + /* + * We don't perform any writes to the cluster so there won't be anything + * to clean up. Also, our cleanup code is really only compatible with + * *write* compatible versions but this runs with *index* compatible + * versions. + */ + return true; + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java deleted file mode 100644 index 9b2aebb2e4c8d..0000000000000 --- a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.qa.verify_version_constants; - -import org.apache.lucene.tests.util.LuceneTestCase; -import org.elasticsearch.Version; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.ObjectPath; - -import java.io.IOException; -import java.text.ParseException; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.lessThan; - -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/97736") -public class VerifyVersionConstantsIT extends ESRestTestCase { - - public void testLuceneVersionConstant() throws IOException, ParseException { - Response response = client().performRequest(new Request("GET", "/")); - assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); - ObjectPath objectPath = ObjectPath.createFromResponse(response); - - String luceneVersionString = objectPath.evaluate("version.lucene_version").toString(); - org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString); - - IndexVersion indexVersion; - Object indexVersionString = objectPath.evaluate("version.index_version"); - if (indexVersionString != null) { - indexVersion = IndexVersion.fromId(Integer.parseInt(indexVersionString.toString())); - } else { - String elasticsearchVersionString = objectPath.evaluate("version.number").toString(); - Version elasticsearchVersion = Version.fromString(elasticsearchVersionString.replace("-SNAPSHOT", "")); - assertThat(elasticsearchVersion, lessThan(Version.V_8_10_0)); - indexVersion = IndexVersion.fromId(elasticsearchVersion.id); - } - - assertThat(indexVersion.luceneVersion(), equalTo(luceneVersion)); - } - - @Override - public boolean preserveClusterUponCompletion() { - /* - * We don't perform any writes to the cluster so there won't be anything - * to clean up. Also, our cleanup code is really only compatible with - * *write* compatible versions but this runs with *index* compatible - * versions. - */ - return true; - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } -} diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index f0df33877a965..49f2415fac392 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -14,7 +14,7 @@ restResources { // REST API specifications are published under the Apache 2.0 License ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) -licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) +licenseFile.set(layout.settingsDirectory.file('licenses/APACHE-LICENSE-2.0.txt').asFile) configurations { // configuration to make use by external yaml rest test plugin in our examples @@ -254,6 +254,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode") task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode") task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode") + task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed") task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0") task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions") task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions") @@ -275,4 +276,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> task.skipTest("search.vectors/130_knn_query_nested_search/nested kNN search inner_hits size > 1", "waiting for #118774 backport") task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport") task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport") + task.skipTest("migration/10_get_feature_upgrade_status/Get feature upgrade status", "Moved to plugin") + task.skipTest("migration/20_post_feature_upgrade/Get feature upgrade status", "Moved to plugin") + task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist") }) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json index 7c929dca1370f..d31188fe7f7fc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json @@ -18,17 +18,6 @@ ] } ] - }, - "params":{ - "help":{ - "type":"boolean", - "description":"Return help information", - "default":false - }, - "s":{ - "type":"list", - "description":"Comma-separated list of column names or column aliases to sort by" - } } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json index 7fe66ea3ba887..eee7e5a5c4b29 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json @@ -36,6 +36,14 @@ "type":"string", "description":"a short version of the Accept header, e.g. json, yaml" }, + "local":{ + "type":"boolean", + "description":"Return local information, do not retrieve the state from master node (default: false)" + }, + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, "bytes":{ "type":"enum", "description":"The unit in which to display byte values", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json index 6969a1c1e595a..e7329cf8dbbf0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json @@ -70,6 +70,16 @@ "type":"boolean", "description":"Verbose mode. Display column headers", "default":false + }, + "timeout":{ + "type":"time", + "default":"30s", + "description":"Period to wait for a response. If no response is received before the timeout expires, the request fails and returns an error." + }, + "wait_for_completion":{ + "type":"boolean", + "default":false, + "description":"If `true`, the request blocks until the task has completed." } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json index 7e4afb14e06e6..492a47c19a19b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json @@ -37,6 +37,11 @@ "description":"Whether the index template should only be added if new or can also replace an existing one", "default":false }, + "cause":{ + "type":"string", + "description": "User defined reason for creating/updating the index template", + "default":"" + }, "master_timeout":{ "type":"time", "description":"Specify timeout for connection to master" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_cluster.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_cluster.json index 8af2dde4f8032..c41233664de0e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_cluster.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_cluster.json @@ -1,55 +1,56 @@ { - "indices.resolve_cluster":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-resolve-cluster-api.html", - "description":"Resolves the specified index expressions to return information about each cluster, including the local cluster, if included." + "indices.resolve_cluster": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-resolve-cluster-api.html", + "description": "Resolves the specified index expressions to return information about each cluster. If no index expression is provided, this endpoint will return information about all the remote clusters that are configured on the local cluster." }, - "stability":"stable", - "visibility":"public", - "headers":{ - "accept": [ "application/json"] + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] }, - "url":{ - "paths":[ + "url": { + "paths": [ { - "path":"/_resolve/cluster/{name}", - "methods":[ - "GET" - ], - "parts":{ - "name":{ - "type":"list", - "description":"A comma-separated list of cluster:index names or wildcard expressions" + "path": "/_resolve/cluster", + "methods": ["GET"] + }, + { + "path": "/_resolve/cluster/{name}", + "methods": ["GET"], + "parts": { + "name": { + "type": "list", + "description": "A comma-separated list of cluster:index names or wildcard expressions" } } } ] }, - "params":{ - "ignore_unavailable":{ - "type":"boolean", - "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)" + "params": { + "ignore_unavailable": { + "type": "boolean", + "description": "Whether specified concrete indices should be ignored when unavailable (missing or closed). Only allowed when providing an index expression." + }, + "ignore_throttled": { + "type": "boolean", + "description": "Whether specified concrete, expanded or aliased indices should be ignored when throttled. Only allowed when providing an index expression." }, - "ignore_throttled":{ - "type":"boolean", - "description":"Whether specified concrete, expanded or aliased indices should be ignored when throttled" + "allow_no_indices": { + "type": "boolean", + "description": "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified). Only allowed when providing an index expression." }, - "allow_no_indices":{ - "type":"boolean", - "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" + "expand_wildcards": { + "type": "enum", + "options": ["open", "closed", "hidden", "none", "all"], + "default": "open", + "description": "Whether wildcard expressions should get expanded to open or closed indices (default: open). Only allowed when providing an index expression." }, - "expand_wildcards":{ - "type":"enum", - "options":[ - "open", - "closed", - "hidden", - "none", - "all" - ], - "default":"open", - "description":"Whether wildcard expressions should get expanded to open or closed indices (default: open)" + "timeout": { + "type": "time", + "description": "The maximum time to wait for remote clusters to respond" } } } } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.chat_completion_unified.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.chat_completion_unified.json new file mode 100644 index 0000000000000..98854625d0471 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.chat_completion_unified.json @@ -0,0 +1,37 @@ +{ + "inference.chat_completion_unified": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/chat-completion-inference.html", + "description": "Perform chat completion inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "text/event-stream" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_inference/chat_completion/{inference_id}/_stream", + "methods": [ + "POST" + ], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.completion.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.completion.json new file mode 100644 index 0000000000000..6c753e59e3434 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.completion.json @@ -0,0 +1,37 @@ +{ + "inference.completion": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", + "description": "Perform completion inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_inference/completion/{inference_id}", + "methods": [ + "POST" + ], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json index 745136848786c..cb4eee007a246 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.delete.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-inference-api.html", "description": "Delete an inference endpoint" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json index 7b7aa0f56fcbc..8887d9d0a1ebe 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get.json @@ -1,47 +1,49 @@ { - "inference.get":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-inference-api.html", - "description":"Get an inference endpoint" + "inference.get": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-inference-api.html", + "description": "Get an inference endpoint" }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "application/json"] + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ] }, - "url":{ - "paths":[ + "url": { + "paths": [ { - "path":"/_inference", - "methods":[ + "path": "/_inference", + "methods": [ "GET" ] }, { - "path":"/_inference/{inference_id}", - "methods":[ + "path": "/_inference/{inference_id}", + "methods": [ "GET" ], - "parts":{ - "inference_id":{ - "type":"string", - "description":"The inference Id" + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" } } }, { - "path":"/_inference/{task_type}/{inference_id}", - "methods":[ + "path": "/_inference/{task_type}/{inference_id}", + "methods": [ "GET" ], - "parts":{ - "task_type":{ - "type":"string", - "description":"The task type" + "parts": { + "task_type": { + "type": "string", + "description": "The task type" }, - "inference_id":{ - "type":"string", - "description":"The inference Id" + "inference_id": { + "type": "string", + "description": "The inference Id" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json index 3195476ce1e9e..bf1282dfaaef7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.inference.json @@ -1,49 +1,45 @@ { - "inference.inference":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", - "description":"Perform inference" + "inference.inference": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", + "description": "Perform inference" }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "application/json"], + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], "content_type": ["application/json"] }, - "url":{ - "paths":[ + "url": { + "paths": [ { - "path":"/_inference/{inference_id}", - "methods":[ - "POST" - ], - "parts":{ - "inference_id":{ - "type":"string", - "description":"The inference Id" + "path": "/_inference/{inference_id}", + "methods": ["POST"], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" } } }, { - "path":"/_inference/{task_type}/{inference_id}", - "methods":[ - "POST" - ], - "parts":{ - "task_type":{ - "type":"string", - "description":"The task type" + "path": "/_inference/{task_type}/{inference_id}", + "methods": ["POST"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" }, - "inference_id":{ - "type":"string", - "description":"The inference Id" + "inference_id": { + "type": "string", + "description": "The inference Id" } } } ] }, - "body":{ - "description":"The inference payload" + "body": { + "description": "The inference payload" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json index 9ff5ff4b80c58..4879007724450 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put.json @@ -1,49 +1,53 @@ { - "inference.put":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-inference-api.html", - "description":"Configure an inference endpoint for use in the Inference API" + "inference.put": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-inference-api.html", + "description": "Configure an inference endpoint for use in the Inference API" }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "application/json"], - "content_type": ["application/json"] + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] }, - "url":{ - "paths":[ + "url": { + "paths": [ { - "path":"/_inference/{inference_id}", - "methods":[ + "path": "/_inference/{inference_id}", + "methods": [ "PUT" ], - "parts":{ - "inference_id":{ - "type":"string", - "description":"The inference Id" + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" } } }, { - "path":"/_inference/{task_type}/{inference_id}", - "methods":[ + "path": "/_inference/{task_type}/{inference_id}", + "methods": [ "PUT" ], - "parts":{ - "task_type":{ - "type":"string", - "description":"The task type" + "parts": { + "task_type": { + "type": "string", + "description": "The task type" }, - "inference_id":{ - "type":"string", - "description":"The inference Id" + "inference_id": { + "type": "string", + "description": "The inference Id" } } } ] }, - "body":{ - "description":"The inference endpoint's task and service settings" + "body": { + "description": "The inference endpoint's task and service settings" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_alibabacloud.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_alibabacloud.json new file mode 100644 index 0000000000000..b39d5abe97ca2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_alibabacloud.json @@ -0,0 +1,35 @@ +{ + "inference.put_alibabacloud": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-alibabacloud-ai-search.html", + "description": "Configure an AlibabaCloud AI Search inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{alibabacloud_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "alibabacloud_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_amazonbedrock.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_amazonbedrock.json new file mode 100644 index 0000000000000..266a1800a360e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_amazonbedrock.json @@ -0,0 +1,35 @@ +{ + "inference.put_amazonbedrock": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-amazon-bedrock.html", + "description": "Configure an Amazon Bedrock inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{amazonbedrock_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "amazonbedrock_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_anthropic.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_anthropic.json new file mode 100644 index 0000000000000..dce56157c7d0e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_anthropic.json @@ -0,0 +1,35 @@ +{ + "inference.put_anthropic": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-anthropic.html", + "description": "Configure an Anthropic inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{anthropic_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "anthropic_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_azureaistudio.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_azureaistudio.json new file mode 100644 index 0000000000000..00de83eca7ce2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_azureaistudio.json @@ -0,0 +1,35 @@ +{ + "inference.put_azureaistudio": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-azure-ai-studio.html", + "description": "Configure an Azure AI Studio inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{azureaistudio_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "azureaistudio_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_azureopenai.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_azureopenai.json new file mode 100644 index 0000000000000..8739adb1f5fd1 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_azureopenai.json @@ -0,0 +1,35 @@ +{ + "inference.put_azureopenai": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-azure-openai.html", + "description": "Configure an Azure OpenAI inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{azureopenai_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "azureopenai_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_cohere.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_cohere.json new file mode 100644 index 0000000000000..a00518f2c5c9e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_cohere.json @@ -0,0 +1,35 @@ +{ + "inference.put_cohere": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-cohere.html", + "description": "Configure a Cohere inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{cohere_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "cohere_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_elasticsearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_elasticsearch.json new file mode 100644 index 0000000000000..0c326b79e93b8 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_elasticsearch.json @@ -0,0 +1,35 @@ +{ + "inference.put_elasticsearch": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elasticsearch.html", + "description": "Configure an Elasticsearch inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{elasticsearch_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "elasticsearch_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_elser.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_elser.json new file mode 100644 index 0000000000000..e601e6c8c3bca --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_elser.json @@ -0,0 +1,39 @@ +{ + "inference.put_elser": { + "deprecated" : { + "version" : "8.16.0", + "description" : "The elser service is deprecated. Use the Elasticsearch inference integration instead, with model_id included in the service_settings." + }, + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elser.html", + "description": "Configure an ELSER inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{elser_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "elser_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_googleaistudio.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_googleaistudio.json new file mode 100644 index 0000000000000..4574626b61c00 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_googleaistudio.json @@ -0,0 +1,35 @@ +{ + "inference.put_googleaistudio": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-google-ai-studio.html", + "description": "Configure a Google AI Studio inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{googleaistudio_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "googleaistudio_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_googlevertexai.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_googlevertexai.json new file mode 100644 index 0000000000000..6068d4cbc91ba --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_googlevertexai.json @@ -0,0 +1,35 @@ +{ + "inference.put_googlevertexai": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-google-vertex-ai.html", + "description": "Configure a Google Vertex AI inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{googlevertexai_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "googlevertexai_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_hugging_face.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_hugging_face.json new file mode 100644 index 0000000000000..76965d61ba839 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_hugging_face.json @@ -0,0 +1,35 @@ +{ + "inference.put_hugging_face": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-hugging-face.html", + "description": "Configure a HuggingFace inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{huggingface_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "huggingface_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_jinaai.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_jinaai.json new file mode 100644 index 0000000000000..80af2a69c9f1c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_jinaai.json @@ -0,0 +1,35 @@ +{ + "inference.put_jinaai": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-jinaai.html", + "description": "Configure a JinaAI inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{jinaai_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "jinaai_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_mistral.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_mistral.json new file mode 100644 index 0000000000000..97633b233ce83 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_mistral.json @@ -0,0 +1,35 @@ +{ + "inference.put_mistral": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-mistral.html", + "description": "Configure a Mistral inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{mistral_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "mistral_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_openai.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_openai.json new file mode 100644 index 0000000000000..5405206c8cacc --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_openai.json @@ -0,0 +1,35 @@ +{ + "inference.put_openai": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-openai.html", + "description": "Configure an OpenAI inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{openai_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "openai_inference_id": { + "type": "string", + "description": "The inference ID" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_voyageai.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_voyageai.json new file mode 100644 index 0000000000000..79a1016560f1a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_voyageai.json @@ -0,0 +1,35 @@ +{ + "inference.put_voyageai": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/inference-apis.html", + "description": "Configure a VoyageAI inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{voyageai_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "voyageai_inference_id": { + "type": "string", + "description": "The inference ID" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_watsonx.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_watsonx.json new file mode 100644 index 0000000000000..db5a42d504b7a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_watsonx.json @@ -0,0 +1,35 @@ +{ + "inference.put_watsonx": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-watsonx-ai.html", + "description": "Configure a Watsonx inference endpoint" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{task_type}/{watsonx_inference_id}", + "methods": ["PUT"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "watsonx_inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.rerank.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.rerank.json new file mode 100644 index 0000000000000..c08a51a8b9b98 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.rerank.json @@ -0,0 +1,37 @@ +{ + "inference.rerank": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", + "description": "Perform reranking inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_inference/rerank/{inference_id}", + "methods": [ + "POST" + ], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.sparse_embedding.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.sparse_embedding.json new file mode 100644 index 0000000000000..90ebb6e6dc4c2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.sparse_embedding.json @@ -0,0 +1,37 @@ +{ + "inference.sparse_embedding": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", + "description": "Perform sparse embedding inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_inference/sparse_embedding/{inference_id}", + "methods": [ + "POST" + ], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_completion.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_completion.json new file mode 100644 index 0000000000000..b4eddb1641233 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_completion.json @@ -0,0 +1,37 @@ +{ + "inference.stream_completion": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/post-stream-inference-api.html", + "description": "Perform streaming completion inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "text/event-stream" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_inference/completion/{inference_id}/_stream", + "methods": [ + "POST" + ], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json deleted file mode 100644 index 32b4b2f311837..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "inference.stream_inference":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-stream-inference-api.html", - "description":"Perform streaming inference" - }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "text/event-stream"], - "content_type": ["application/json"] - }, - "url":{ - "paths":[ - { - "path":"/_inference/{inference_id}/_stream", - "methods":[ - "POST" - ], - "parts":{ - "inference_id":{ - "type":"string", - "description":"The inference Id" - } - } - }, - { - "path":"/_inference/{task_type}/{inference_id}/_stream", - "methods":[ - "POST" - ], - "parts":{ - "task_type":{ - "type":"string", - "description":"The task type" - }, - "inference_id":{ - "type":"string", - "description":"The inference Id" - } - } - } - ] - }, - "body":{ - "description":"The inference payload" - } - } -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.text_embedding.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.text_embedding.json new file mode 100644 index 0000000000000..309a1d80b7416 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.text_embedding.json @@ -0,0 +1,37 @@ +{ + "inference.text_embedding": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/post-inference-api.html", + "description": "Perform text embedding inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_inference/text_embedding/{inference_id}", + "methods": [ + "POST" + ], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.unified_inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.unified_inference.json deleted file mode 100644 index 84182d19f8825..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.unified_inference.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "inference.unified_inference": { - "documentation": { - "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/unified-inference-api.html", - "description": "Perform inference using the Unified Schema" - }, - "stability": "stable", - "visibility": "public", - "headers": { - "accept": ["text/event-stream"], - "content_type": ["application/json"] - }, - "url": { - "paths": [ - { - "path": "/_inference/{inference_id}/_unified", - "methods": ["POST"], - "parts": { - "inference_id": { - "type": "string", - "description": "The inference Id" - } - } - }, - { - "path": "/_inference/{task_type}/{inference_id}/_unified", - "methods": ["POST"], - "parts": { - "task_type": { - "type": "string", - "description": "The task type" - }, - "inference_id": { - "type": "string", - "description": "The inference Id" - } - } - } - ] - }, - "body": { - "description": "The inference payload" - } - } -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json index d032da3b3cda2..3abf072e2efe5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json @@ -75,6 +75,10 @@ "options": ["starting", "started", "fully_allocated"], "default": "started" } + }, + "body":{ + "description": "The settings for the trained model deployment", + "required": false } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.allocation/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.allocation/10_basic.yml index 0d660cbb6b048..6e7d14e4dc7aa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.allocation/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.allocation/10_basic.yml @@ -295,7 +295,7 @@ ( [-\w.]+ \s+ [-\w.]+ \s+ [-\w.]+ \s+ - [\w]+ + [-\w.]+ \n )+ $/ diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml index 8645c91a51ad3..378bc909ca3c5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -214,3 +214,68 @@ index.mode: lookup index.number_of_shards: 2 +--- +"Poorly formatted request returns bad_request": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ poorly_formatted_bad_request ] + reason: "requires poorly_formatted_bad_request bug fix" + + - do: + catch: bad_request + indices.create: + index: test_index + body: + mappings: "bad mappings" + + - do: + catch: bad_request + indices.create: + index: test_index + body: + mappings: + properties: "bad properties" + + - do: + catch: bad_request + indices.create: + index: test_index + body: + settings: "bad settings" + + - do: + catch: bad_request + indices.create: + index: test_index + body: + aliases: "bad alias" +--- +"Create index with hunspell missing dict": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ hunspell_dict_400 ] + reason: "bugfix 'hunspell_dict_400' capability required" + + - do: + catch: bad_request + indices.create: + index: bad_hunspell_index + body: + settings: + analysis: + analyzer: + en: + tokenizer: standard + filter: + - my_en_US_dict_stemmer + filter: + my_en_US_dict_stemmer: + type: hunspell + locale: en_US + dedup: false diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 90c0bc87c8ea3..97e6e96dc16f2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -2118,3 +2118,143 @@ create index with use_synthetic_source: flush: false - gt: { test.store_size_in_bytes: 0 } - is_false: test.fields._recovery_source +--- +"Nested synthetic source with indexed dense vectors": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ synthetic_nested_dense_vector_bug_fix ] + reason: "Requires synthetic source bugfix for dense vectors in nested objects" + - do: + indices.create: + index: nested_dense_vector_synthetic_test + body: + mappings: + properties: + parent: + type: nested + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + text: + type: text + settings: + index: + mapping: + source: + mode: synthetic + - do: + index: + index: nested_dense_vector_synthetic_test + id: 0 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ],"text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 1 + refresh: true + body: { "parent": [ { "text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 2 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ] }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + + - do: + search: + index: nested_dense_vector_synthetic_test + body: + query: + match_all: {} + + - match: { hits.hits.0._source.parent.0.vector: [ 1.0, 2.0 ] } + - match: { hits.hits.0._source.parent.0.text: "foo" } + - match: { hits.hits.0._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.0._source.parent.1.text: "bar" } + - is_false: hits.hits.1._source.parent.0.vector + - match: { hits.hits.1._source.parent.0.text: "foo" } + - match: { hits.hits.1._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.1._source.parent.1.text: "bar" } + - match: {hits.hits.2._source.parent.0.vector: [ 1.0, 2.0 ] } + - is_false: hits.hits.2._source.parent.0.text + - match: { hits.hits.2._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.2._source.parent.1.text: "bar" } +--- +"Nested synthetic source with un-indexed dense vectors": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ synthetic_nested_dense_vector_bug_fix ] + reason: "Requires synthetic source bugfix for dense vectors in nested objects" + - do: + indices.create: + index: nested_dense_vector_synthetic_test + body: + mappings: + properties: + parent: + type: nested + properties: + vector: + type: dense_vector + index: false + text: + type: text + settings: + index: + mapping: + source: + mode: synthetic + - do: + index: + index: nested_dense_vector_synthetic_test + id: 0 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ],"text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 1 + refresh: true + body: { "parent": [ { "text": "foo" }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + - do: + index: + index: nested_dense_vector_synthetic_test + id: 2 + refresh: true + body: { "parent": [ { "vector": [ 1, 2 ] }, { "vector": [ 2, 2 ], "text": "bar" } ] } + + + - do: + search: + index: nested_dense_vector_synthetic_test + body: + query: + match_all: {} + + - match: { hits.hits.0._source.parent.0.vector: [ 1.0, 2.0 ] } + - match: { hits.hits.0._source.parent.0.text: "foo" } + - match: { hits.hits.0._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.0._source.parent.1.text: "bar" } + - is_false: hits.hits.1._source.parent.0.vector + - match: { hits.hits.1._source.parent.0.text: "foo" } + - match: { hits.hits.1._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.1._source.parent.1.text: "bar" } + - match: {hits.hits.2._source.parent.0.vector: [ 1.0, 2.0 ] } + - is_false: hits.hits.2._source.parent.0.text + - match: { hits.hits.2._source.parent.1.vector: [ 2.0, 2.0 ] } + - match: { hits.hits.2._source.parent.1.text: "bar" } + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml index ba341e0d220e1..75d7a38f6ef4a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml @@ -41,7 +41,7 @@ setup: - do: indices.resolve_cluster: name: '*' - expand_wildcards: [closed] + expand_wildcards: closed - match: {(local).connected: true} - match: {(local).skip_unavailable: false} @@ -67,7 +67,7 @@ setup: - do: indices.resolve_cluster: name: 'index2*' - expand_wildcards: [open,closed] + expand_wildcards: open,closed - match: {(local).connected: true} - match: {(local).skip_unavailable: false} @@ -77,7 +77,7 @@ setup: - do: indices.resolve_cluster: name: 'index2*' - expand_wildcards: [closed] + expand_wildcards: closed - match: {(local).connected: true} - match: {(local).skip_unavailable: false} @@ -117,7 +117,7 @@ setup: - do: indices.resolve_cluster: name: 'my_alias2,doesnotexist*' - expand_wildcards: [all] + expand_wildcards: all - match: {(local).connected: true} - match: {(local).skip_unavailable: false} @@ -143,10 +143,10 @@ setup: - do: indices.resolve_cluster: name: '*' - expand_wildcards: [all] - ignore_unavailable: [true] - ignore_throttled: [true] - allow_no_indices: [true] + expand_wildcards: all + ignore_unavailable: true + ignore_throttled: true + allow_no_indices: true allowed_warnings: - "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices." @@ -159,10 +159,10 @@ setup: - do: indices.resolve_cluster: name: '*' - expand_wildcards: [open] - ignore_unavailable: [false] - ignore_throttled: [false] - allow_no_indices: [false] + expand_wildcards: open + ignore_unavailable: false + ignore_throttled: false + allow_no_indices: false allowed_warnings: - "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices." @@ -172,3 +172,14 @@ setup: - is_false: (local).error # should not be present - exists: (local).version.number +--- +"Resolve cluster with no index expression": + - requires: + cluster_features: ["gte_v8.18.0"] + reason: "resolve cluster with no index expression introduced in 8.18" + + - do: + indices.resolve_cluster: + timeout: 400s + + - is_false: (local).error # should not be present - body should be empty since no remotes configured diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.rollover/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.rollover/10_basic.yml index a53365721cf0c..222035174b8bb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.rollover/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.rollover/10_basic.yml @@ -183,3 +183,19 @@ min_age: "0s" min_docs: 1 - match: { error.reason: "Validation Failed: 1: at least one max_* rollover condition must be set when using min_* conditions;" } + +--- +"Rolling over an unknown target should return 404": + - requires: + capabilities: + - method: POST + path: /{index}/_rollover + capabilities: ['return-404-on-missing-target'] + test_runner_features: [capabilities] + reason: Rollover used to return a 400, then it briefly returned a 500 due to an NPE, now it properly returns a 404 + + - do: + catch: missing + indices.rollover: + alias: "non_existent" + - match: {error.reason: "rollover target [non_existent] does not exist"} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml index d732fb084db3d..9b3291e19c5fd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml @@ -115,12 +115,70 @@ setup: - match: {hits.hits.0.highlight.field2.0: "The quick brown fox went to the forest and saw another fox."} --- -"Plain highlighter with max_analyzed_offset < 0 should FAIL": +"Plain highlighter on a field WITH OFFSETS exceeding index.highlight.max_analyzed_offset with max_analyzed_offset=0 should FAIL": + + - requires: + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [ highlight_max_analyzed_offset_default ] + reason: Behavior of max_analyzed_offset query param changed in 8.18. + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + index: test1 + body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}, "max_analyzed_offset": 0}} + - match: { status: 400 } + - match: { error.root_cause.0.type: "x_content_parse_exception" } + - match: { error.caused_by.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "[max_analyzed_offset] must be a positive integer, or -1" } + +--- +"Plain highlighter on a field WITH OFFSETS exceeding index.highlight.max_analyzed_offset with max_analyzed_offset=1 should SUCCEED": - requires: cluster_features: ["gte_v7.12.0"] reason: max_analyzed_offset query param added in 7.12.0 + - do: + search: + rest_total_hits_as_int: true + index: test1 + body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}, "max_analyzed_offset": 1}} + - match: { hits.hits.0.highlight: null } + +--- +"Plain highlighter with max_analyzed_offset = -1 default to index analyze offset should SUCCEED": + + - requires: + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [ highlight_max_analyzed_offset_default ] + reason: Behavior of max_analyzed_offset query param changed in 8.18. + + - do: + search: + rest_total_hits_as_int: true + index: test1 + body: {"query" : {"match" : {"field2" : "fox"}}, "highlight" : {"type" : "plain", "fields" : {"field2" : {}}, "max_analyzed_offset": -1}} + - match: {hits.hits.0.highlight.field2.0: "The quick brown fox went to the forest and saw another fox."} + +--- +"Plain highlighter with max_analyzed_offset < -1 should FAIL": + + - requires: + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [ highlight_max_analyzed_offset_default ] + reason: Behavior of max_analyzed_offset query param changed in 8.18. + - do: catch: bad_request search: @@ -130,4 +188,4 @@ setup: - match: { status: 400 } - match: { error.root_cause.0.type: "x_content_parse_exception" } - match: { error.caused_by.type: "illegal_argument_exception" } - - match: { error.caused_by.reason: "[max_analyzed_offset] must be a positive integer" } + - match: { error.caused_by.reason: "[max_analyzed_offset] must be a positive integer, or -1" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml index 9cb17af41dfbf..481f7879e0c4b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml @@ -596,3 +596,28 @@ setup: - match: { hits.hits.0._score: $knn_score0 } - match: { hits.hits.1._score: $knn_score1 } - match: { hits.hits.2._score: $knn_score2 } +--- +"Updating dim to null is not allowed": + - requires: + cluster_features: "mapper.npe_on_dims_update_fix" + reason: "dims update fix" + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + - do: + catch: bad_request + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml index 3f81c0044d170..4c9d1ef881c6d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml @@ -68,7 +68,7 @@ setup: -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, - -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] # Flush in order to provoke a merge later - do: indices.flush: @@ -78,6 +78,9 @@ setup: indices.forcemerge: index: bbq_hnsw max_num_segments: 1 + + - do: + indices.refresh: { } --- "Test knn search": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml index 0e0180e58fd96..71865de6e0a1c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml @@ -60,8 +60,15 @@ setup: another_vector: [-0.5, 11.0, 0, 12] - do: - indices.refresh: {} + indices.flush: { } + # For added test reliability, pending the resolution of https://github.com/elastic/elasticsearch/issues/109416. + - do: + indices.forcemerge: + max_num_segments: 1 + index: int4_flat + - do: + indices.refresh: {} --- "kNN search only": - do: @@ -195,13 +202,14 @@ setup: num_candidates: 3 k: 3 field: vector - similarity: 10.3 + # Set high allowed similarity, reduce once we can update underlying quantization algo + similarity: 110 query_vector: [-0.5, 90.0, -10, 14.8] - - length: {hits.hits: 1} + - is_true: hits.hits.0 - - match: {hits.hits.0._id: "2"} - - match: {hits.hits.0.fields.name.0: "moose.jpg"} + #- match: {hits.hits.0._id: "2"} + #- match: {hits.hits.0.fields.name.0: "moose.jpg"} --- "Vector similarity with filter only": - do: @@ -213,7 +221,8 @@ setup: num_candidates: 3 k: 3 field: vector - similarity: 11 + # Set high allowed similarity, reduce once we can update underlying quantization algo + similarity: 110 query_vector: [-0.5, 90.0, -10, 14.8] filter: {"term": {"name": "moose.jpg"}} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml index e9bfffb8da604..02dcf978040c0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml @@ -149,3 +149,39 @@ setup: to: 2023 include_lower: false include_upper: false + +--- +"test bad dates in range - past": + - requires: + cluster_features: [ "mapper.range.invalid_date_fix" ] + reason: "Fix for invalid date required" + - do: + catch: /illegal_argument_exception/ + search: + index: dates + body: + sort: field + query: + range: + date: + gte: -522000000 + lte: 2023 + format: date_optional_time + +--- +"test bad dates in range - future": + - requires: + cluster_features: [ "mapper.range.invalid_date_fix" ] + reason: "Fix for invalid date required" + - do: + catch: /illegal_argument_exception/ + search: + index: dates + body: + sort: field + query: + range: + date: + gte: 2020 + lte: 522000000 + format: date_optional_time diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml index 5a60f76f6da2c..84e0d5b3e524a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml @@ -68,3 +68,10 @@ setup: wait_for_completion: false - match: { acknowledged: true } + + # now create another snapshot just to ensure that the async delete finishes before the test cleanup runs: + - do: + snapshot.create: + repository: test_repo_create_1 + snapshot: barrier_snapshot + wait_for_completion: true diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml index 7f545b466e65f..3baeb39e410c8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml @@ -315,3 +315,160 @@ setup: indices.stats: { index: test_index } - length: { indices: 0 } + +--- +"Load index with non existent synonyms set": + - requires: + cluster_features: [ index.synonyms_set_lenient_on_non_existing ] + reason: "requires synonyms_set_lenient_on_non_existing bug fix" + - do: + indices.create: + index: test_index + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + analysis: + filter: + my_synonym_filter: + type: synonym + synonyms_set: set1 + updateable: true + analyzer: + my_analyzer: + type: custom + tokenizer: whitespace + filter: [ lowercase, my_synonym_filter ] + mappings: + properties: + my_field: + type: text + search_analyzer: my_analyzer + + - match: { acknowledged: true } + - match: { shards_acknowledged: true } + + - do: + indices.stats: { index: test_index } + + - match: { indices.test_index.health: "green" } + + # Synonyms are not applied + - do: + indices.analyze: + index: test_index + body: + analyzer: my_analyzer + text: foo + + - length: { tokens: 1 } + - match: { tokens.0.token: foo } + + + # Create synonyms set and check synonyms are applied + - do: + synonyms.put_synonym: + id: set1 + body: + synonyms_set: + synonyms: "foo => bar, baz" + + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms + wait_for_status: green + + + - do: + indices.stats: { index: test_index } + + - match: { indices.test_index.health: "green" } + + # Synonyms are applied + - do: + indices.analyze: + index: test_index + body: + analyzer: my_analyzer + text: foo + + - length: { tokens: 2 } + +--- +"Load index with non existent synonyms set and lenient set to false": + - requires: + test_runner_features: [ allowed_warnings ] + + - do: + indices.create: + index: test_index + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + analysis: + filter: + my_synonym_filter: + type: synonym + synonyms_set: set1 + updateable: true + lenient: false + analyzer: + my_analyzer: + type: custom + tokenizer: whitespace + filter: [ lowercase, my_synonym_filter ] + mappings: + properties: + my_field: + type: text + search_analyzer: my_analyzer + + - match: { acknowledged: true } + - match: { shards_acknowledged: false } + + - do: + indices.stats: { index: test_index } + + - length: { indices: 0 } + + # Create synonyms set and check synonyms are applied + - do: + synonyms.put_synonym: + id: set1 + body: + synonyms_set: + synonyms: "foo => bar, baz" + + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms + # BWC tests won't be able to get green - synonyms index is 0-all, so some shards won't be recovered in older nodes + wait_for_status: yellow + + - do: + # Warning issued in previous versions + allowed_warnings: + - "The [state] field in the response to the reroute API is deprecated and will be removed in a future version. Specify ?metric=none to adopt the future behaviour." + cluster.reroute: + retry_failed: true + + - do: + cluster.health: + index: test_index + wait_for_status: green + + # Synonyms are applied + - do: + indices.analyze: + index: test_index + body: + analyzer: my_analyzer + text: foo + + - length: { tokens: 2 } + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml index 90c35cc8f4888..fbbd42ad590dc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml @@ -168,34 +168,6 @@ setup: query: hola - match: { hits.total.value: 1 } ---- -"Fail loading synonyms from index if synonyms_set doesn't exist": - - do: - indices.create: - index: another_index - body: - settings: - index: - number_of_shards: 1 - analysis: - filter: - my_synonym_filter: - type: synonym - synonyms_set: set_missing - updateable: true - analyzer: - my_analyzer: - type: custom - tokenizer: standard - filter: [ lowercase, my_synonym_filter ] - mappings: - properties: - my_field: - type: text - search_analyzer: my_analyzer - - match: { acknowledged: true } - - match: { shards_acknowledged: false } - --- "Load empty synonyms set from index for an analyzer": - do: diff --git a/server/build.gradle b/server/build.gradle index 2a27e653b4453..d8267369fd1c9 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -82,6 +82,7 @@ dependencies { } internalClusterTestImplementation(project(':modules:reindex')) internalClusterTestImplementation(project(':modules:mapper-extras')) + internalClusterTestImplementation(project(':modules:data-streams')) } spotless { @@ -133,7 +134,7 @@ def generatePluginsList = tasks.register("generatePluginsList") { sourceSets.main.output.dir(generatedResourcesDir) sourceSets.main.compiledBy(generateModulesList, generatePluginsList) -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' @@ -146,18 +147,30 @@ if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.insecure_network_trace_enabled', 'true' + filter { + excludeTestsMatching("*.TransportServiceHandshakeTests.testAcceptsMismatchedServerlessBuildHash") + } excludes << '**/IndexSettingsOverrideTests.class' } -TaskProvider indexSettingsOverrideTest = tasks.register("indexSettingsOverrideTest", Test) { +// There are tests rely on system properties to be configured differently. They must run in a separate test job +// since the default does not work for them and configuring the system properties inside the test class/method +// is too late because fields based on the system properties are often initialized statically. +TaskProvider systemPropertiesOverrideTest = tasks.register("systemPropertiesOverrideTest", Test) { include '**/IndexSettingsOverrideTests.class' + include '**/TransportServiceHandshakeTests.class' + filter { + includeTestsMatching("*.TransportServiceHandshakeTests.testAcceptsMismatchedServerlessBuildHash") + includeTestsMatching("*.IndexSettingsOverrideTests.*") + } systemProperty 'es.stateless.allow.index.refresh_interval.override', 'true' + systemProperty 'es.serverless_transport', 'true' classpath = sourceSets.test.runtimeClasspath testClassesDirs = sourceSets.test.output.classesDirs } tasks.named("check").configure { - dependsOn(indexSettingsOverrideTest) + dependsOn(systemPropertiesOverrideTest) } tasks.named("thirdPartyAudit").configure { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index a5aa39f5feb1e..83e79ff7f45a8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -85,7 +85,7 @@ public void testMissingKeystoreFile() throws Exception { final Environment environment = internalCluster().getInstance(Environment.class); final AtomicReference reloadSettingsError = new AtomicReference<>(); // keystore file should be missing for this test case - Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configFile())); + Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configDir())); final int initialReloadCount = mockReloadablePlugin.getReloadCount(); final CountDownLatch latch = new CountDownLatch(1); executeReloadSecureSettings(Strings.EMPTY_ARRAY, emptyPassword(), new ActionListener<>() { @@ -129,10 +129,10 @@ public void testInvalidKeystoreFile() throws Exception { final int initialReloadCount = mockReloadablePlugin.getReloadCount(); // invalid "keystore" file should be present in the config dir try (InputStream keystore = ReloadSecureSettingsIT.class.getResourceAsStream("invalid.txt.keystore")) { - if (Files.exists(environment.configFile()) == false) { - Files.createDirectory(environment.configFile()); + if (Files.exists(environment.configDir()) == false) { + Files.createDirectory(environment.configDir()); } - Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configFile()), StandardCopyOption.REPLACE_EXISTING); + Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configDir()), StandardCopyOption.REPLACE_EXISTING); } final CountDownLatch latch = new CountDownLatch(1); executeReloadSecureSettings(Strings.EMPTY_ARRAY, emptyPassword(), new ActionListener<>() { @@ -363,7 +363,7 @@ public void testInvalidKeyInSettings() throws Exception { try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { keyStoreWrapper.setString(VALID_SECURE_SETTING_NAME, new char[0]); - keyStoreWrapper.save(environment.configFile(), new char[0], false); + keyStoreWrapper.save(environment.configDir(), new char[0], false); } PlainActionFuture actionFuture = new PlainActionFuture<>(); @@ -374,7 +374,7 @@ public void testInvalidKeyInSettings() throws Exception { try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { assertThat(keyStoreWrapper, notNullValue()); keyStoreWrapper.setString("some.setting.that.does.not.exist", new char[0]); - keyStoreWrapper.save(environment.configFile(), new char[0], false); + keyStoreWrapper.save(environment.configDir(), new char[0], false); } actionFuture = new PlainActionFuture<>(); @@ -432,7 +432,7 @@ public void onFailure(Exception e) { private SecureSettings writeEmptyKeystore(Environment environment, char[] password) throws Exception { final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create(); - keyStoreWrapper.save(environment.configFile(), password, false); + keyStoreWrapper.save(environment.configDir(), password, false); return keyStoreWrapper; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java index b10e17b3f4e0f..30669f38ab212 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java @@ -28,7 +28,7 @@ import org.elasticsearch.indices.TestSystemIndexPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.snapshots.SystemIndicesSnapshotIT; +import org.elasticsearch.snapshots.SystemResourceSnapshotIT; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentType; import org.junit.After; @@ -298,7 +298,7 @@ public Collection getSystemIndexDescriptors(Settings sett @Override public String getFeatureName() { - return SystemIndicesSnapshotIT.SystemIndexTestPlugin.class.getSimpleName(); + return SystemResourceSnapshotIT.SystemIndexTestPlugin.class.getSimpleName(); } @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingIT.java new file mode 100644 index 0000000000000..0a29b99ca6fdc --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingIT.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.indices.mapping.put; + +import org.apache.logging.log4j.Level; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.junit.annotations.TestLogging; + +import static org.hamcrest.Matchers.equalTo; + +public class PutMappingIT extends ESSingleNodeTestCase { + + @TestLogging( + reason = "testing DEBUG logging", + value = "org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction:DEBUG" + ) + public void testFailureLogging() { + final var indexName = randomIdentifier(); + createIndex(indexName); + final var fieldName = randomIdentifier(); + safeGet(client().execute(TransportPutMappingAction.TYPE, new PutMappingRequest(indexName).source(fieldName, "type=keyword"))); + MockLog.assertThatLogger( + () -> assertThat( + asInstanceOf( + IllegalArgumentException.class, + safeAwaitFailure( + AcknowledgedResponse.class, + l -> client().execute( + TransportPutMappingAction.TYPE, + new PutMappingRequest(indexName).source(fieldName, "type=long"), + l + ) + ) + ).getMessage(), + equalTo("mapper [" + fieldName + "] cannot be changed from type [keyword] to [long]") + ), + TransportPutMappingAction.class, + new MockLog.SeenEventExpectation( + "failure message", + TransportPutMappingAction.class.getCanonicalName(), + Level.DEBUG, + "failed to put mappings on indices [[" + indexName + ) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java index 90e3196d76378..56ac96d592f49 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java @@ -179,28 +179,35 @@ public void testNodeRemovalFromRedClusterWithTimeout() throws Exception { // make it red! internalCluster().stopNode(node1); ensureRed(indexName); + CountDownLatch stallPrevalidateShardPathActionLatch = new CountDownLatch(1); MockTransportService.getInstance(node2) .addRequestHandlingBehavior(TransportPrevalidateShardPathAction.ACTION_NAME + "[n]", (handler, request, channel, task) -> { logger.info("drop the check shards request"); + safeAwait(stallPrevalidateShardPathActionLatch); + handler.messageReceived(request, channel, task); }); - PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder() - .setNames(node2) - .build(TEST_REQUEST_TIMEOUT) - .masterNodeTimeout(TimeValue.timeValueSeconds(1)) - .timeout(TimeValue.timeValueSeconds(1)); - PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get(); - assertFalse("prevalidation result should return false", resp.getPrevalidation().isSafe()); - String node2Id = getNodeId(node2); - assertThat( - resp.getPrevalidation().message(), - equalTo("cannot prevalidate removal of nodes with the following IDs: [" + node2Id + "]") - ); - assertThat(resp.getPrevalidation().nodes().size(), equalTo(1)); - NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0); - assertThat(nodeResult.name(), equalTo(node2)); - assertFalse(nodeResult.result().isSafe()); - assertThat(nodeResult.result().message(), startsWith("failed contacting the node")); - assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.UNABLE_TO_VERIFY)); + try { + PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder() + .setNames(node2) + .build(TEST_REQUEST_TIMEOUT) + .masterNodeTimeout(TimeValue.timeValueSeconds(1)) + .timeout(TimeValue.timeValueSeconds(1)); + PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get(); + assertFalse("prevalidation result should return false", resp.getPrevalidation().isSafe()); + String node2Id = getNodeId(node2); + assertThat( + resp.getPrevalidation().message(), + equalTo("cannot prevalidate removal of nodes with the following IDs: [" + node2Id + "]") + ); + assertThat(resp.getPrevalidation().nodes().size(), equalTo(1)); + NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0); + assertThat(nodeResult.name(), equalTo(node2)); + assertFalse(nodeResult.result().isSafe()); + assertThat(nodeResult.result().message(), startsWith("failed contacting the node")); + assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.UNABLE_TO_VERIFY)); + } finally { + stallPrevalidateShardPathActionLatch.countDown(); + } } private void ensureRed(String indexName) throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java index c1e68040e075b..9d486abeafc4f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java @@ -39,7 +39,8 @@ public void testThatNonDynamicSettingChangesTakeEffect() throws Exception { * This test makes sure that when non-dynamic settings are updated that they actually take effect (as opposed to just being set * in the cluster state). */ - createIndex("test", Settings.EMPTY); + createIndex("test-1", Settings.EMPTY); + createIndex("test-2", Settings.EMPTY); MetadataUpdateSettingsService metadataUpdateSettingsService = internalCluster().getCurrentMasterNodeInstance( MetadataUpdateSettingsService.class ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 870947db5bd85..e574e1468c942 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -98,7 +98,6 @@ import static org.elasticsearch.index.shard.IndexShardTestCase.closeShardNoCheck; import static org.elasticsearch.index.shard.IndexShardTestCase.getTranslog; import static org.elasticsearch.index.shard.IndexShardTestCase.recoverFromStore; -import static org.elasticsearch.indices.cluster.AbstractIndicesClusterStateServiceTestCase.awaitIndexShardCloseAsyncTasks; import static org.elasticsearch.test.LambdaMatchers.falseWith; import static org.elasticsearch.test.LambdaMatchers.trueWith; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -221,7 +220,7 @@ public void testUpdatePriority() { public void testIndexDirIsDeletedWhenShardRemoved() throws Exception { Environment env = getInstanceFromNode(Environment.class); - Path idxPath = env.sharedDataFile().resolve(randomAlphaOfLength(10)); + Path idxPath = env.sharedDataDir().resolve(randomAlphaOfLength(10)); logger.info("--> idxPath: [{}]", idxPath); Settings idxSettings = Settings.builder().put(IndexMetadata.SETTING_DATA_PATH, idxPath).build(); createIndex("test", idxSettings); @@ -255,7 +254,7 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException { public void testIndexCanChangeCustomDataPath() throws Exception { final String index = "test-custom-data-path"; - final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataFile().resolve(randomAsciiLettersOfLength(10)); + final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataDir().resolve(randomAsciiLettersOfLength(10)); final Path indexDataPath = sharedDataPath.resolve("start-" + randomAsciiLettersOfLength(10)); logger.info("--> creating index [{}] with data_path [{}]", index, indexDataPath); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java index bd58526c61432..153bea7863e1a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -562,7 +562,7 @@ public void testResolvePath() throws Exception { command.findAndProcessShardPath( options, environmentByNodeName.get(nodeName), - environmentByNodeName.get(nodeName).dataFiles(), + environmentByNodeName.get(nodeName).dataDirs(), state, shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath)) ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java index 90326abb381d0..dd3f01a386001 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java @@ -253,7 +253,7 @@ public void testSettingsApplied() throws Exception { FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); - assertTrue(masterFileSettingsService.watching()); + assertBusy(() -> assertTrue(masterFileSettingsService.watching())); assertFalse(dataFileSettingsService.watching()); writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet()); @@ -279,7 +279,7 @@ public void testSettingsAppliedOnStart() throws Exception { FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); - assertTrue(masterFileSettingsService.watching()); + assertBusy(() -> assertTrue(masterFileSettingsService.watching())); assertFalse(dataFileSettingsService.watching()); assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb"); @@ -374,7 +374,7 @@ public void testErrorSaved() throws Exception { FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); - assertTrue(masterFileSettingsService.watching()); + assertBusy(() -> assertTrue(masterFileSettingsService.watching())); assertFalse(dataFileSettingsService.watching()); writeJSONFile(masterNode, testErrorJSON, logger, versionCounter.incrementAndGet()); @@ -470,7 +470,7 @@ public void testSettingsAppliedOnMasterReElection() throws Exception { var savedClusterState = setupClusterStateListener(masterNode); FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); - assertTrue(masterFileSettingsService.watching()); + assertBusy(() -> assertTrue(masterFileSettingsService.watching())); writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet()); assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java index 5f8bc57dcbe09..08971b0573b96 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java @@ -238,7 +238,6 @@ public void testCancelMultiSearch() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99929") public void testCancelFailedSearchWhenPartialResultDisallowed() throws Exception { // Have at least two nodes so that we have parallel execution of two request guaranteed even if max concurrent requests per node // are limited to 1 @@ -262,12 +261,16 @@ public void testCancelFailedSearchWhenPartialResultDisallowed() throws Exception // When the search request executes, block all shards except 1. final List searchShardBlockingPlugins = initSearchShardBlockingPlugin(); AtomicBoolean letOneShardProceed = new AtomicBoolean(); + // Ensure we have at least one task waiting on the latch + CountDownLatch waitingTaskLatch = new CountDownLatch(1); CountDownLatch shardTaskLatch = new CountDownLatch(1); for (SearchShardBlockingPlugin plugin : searchShardBlockingPlugins) { plugin.setRunOnNewReaderContext((ReaderContext c) -> { if (letOneShardProceed.compareAndSet(false, true)) { // Let one shard continue. } else { + // Signal that we have a task waiting on the latch + waitingTaskLatch.countDown(); safeAwait(shardTaskLatch); // Block the other shards. } }); @@ -280,6 +283,9 @@ public void testCancelFailedSearchWhenPartialResultDisallowed() throws Exception plugin.disableBlock(); plugin.setBeforeExecution(() -> { if (oneThreadWillError.compareAndSet(false, true)) { + // wait for some task to get to the latch + safeAwait(waitingTaskLatch); + // then throw the exception throw new IllegalStateException("This will cancel the ContextIndexSearcher.search task"); } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java index ee1aac60da9c1..96f4af5ce1bf4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java @@ -9,36 +9,69 @@ package org.elasticsearch.search; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.MockScriptPlugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.rescore.RescoreContext; +import org.elasticsearch.search.rescore.Rescorer; +import org.elasticsearch.search.rescore.RescorerBuilder; +import org.elasticsearch.search.suggest.SortBy; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.search.suggest.SuggestionSearchContext; +import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.XContentBuilder; +import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.Map; +import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; -import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; -import static org.elasticsearch.search.SearchTimeoutIT.ScriptedTimeoutPlugin.SCRIPT_NAME; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) +@ESIntegTestCase.SuiteScopeTestCase public class SearchTimeoutIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(ScriptedTimeoutPlugin.class); + return Collections.singleton(SearchTimeoutPlugin.class); } @Override @@ -46,75 +79,475 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build(); } - private void indexDocs() { - for (int i = 0; i < 32; i++) { - prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get(); - } - refresh("test"); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98369") - public void testTopHitsTimeout() { - indexDocs(); - SearchResponse searchResponse = prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS)) - .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .get(); - assertThat(searchResponse.isTimedOut(), equalTo(true)); - assertEquals(0, searchResponse.getShardFailures().length); - assertEquals(0, searchResponse.getFailedShards()); - assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); - assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); - assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L)); - assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98053") - public void testAggsTimeout() { - indexDocs(); - SearchResponse searchResponse = prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS)) - .setSize(0) - .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")) - .get(); - assertThat(searchResponse.isTimedOut(), equalTo(true)); - assertEquals(0, searchResponse.getShardFailures().length); - assertEquals(0, searchResponse.getFailedShards()); - assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); - assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); - assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L)); - assertEquals(searchResponse.getHits().getHits().length, 0); - StringTerms terms = searchResponse.getAggregations().get("terms"); - assertEquals(1, terms.getBuckets().size()); - StringTerms.Bucket bucket = terms.getBuckets().get(0); - assertEquals("value", bucket.getKeyAsString()); - assertThat(bucket.getDocCount(), greaterThan(0L)); - } - - public void testPartialResultsIntolerantTimeout() throws Exception { - prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + @Override + protected void setupSuiteScopeCluster() throws Exception { + super.setupSuiteScopeCluster(); + indexRandom(true, "test", randomIntBetween(20, 50)); + } + + /** + * Test the scenario where the query times out before starting to collect documents, verify that partial hits are not returned + */ + public void testTopHitsTimeoutBeforeCollecting() { + // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.SECONDS)) + .setQuery(new BulkScorerTimeoutQuery(false)); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + // timeout happened before we could collect any doc, total hits is 0 and no hits are returned + assertEquals(0, searchResponse.getHits().getTotalHits().value); + assertEquals(0, searchResponse.getHits().getHits().length); + }); + } + + /** + * Test the scenario where the query times out while collecting documents, verify that partial hits results are returned + */ + public void testTopHitsTimeoutWhileCollecting() { + // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.SECONDS)) + .setQuery(new BulkScorerTimeoutQuery(true)); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L)); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + }); + } + + /** + * Test the scenario where the query times out before starting to collect documents, verify that partial aggs results are not returned + */ + public void testAggsTimeoutBeforeCollecting() { + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setSize(0) + // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set + .setTimeout(new TimeValue(10, TimeUnit.SECONDS)) + .setQuery(new BulkScorerTimeoutQuery(false)) + .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertEquals(0, searchResponse.getHits().getTotalHits().value); + assertEquals(0, searchResponse.getHits().getHits().length); + StringTerms terms = searchResponse.getAggregations().get("terms"); + // timeout happened before we could collect any doc, total hits is 0 and no buckets are returned + assertEquals(0, terms.getBuckets().size()); + }); + } + + /** + * Test the scenario where the query times out while collecting documents, verify that partial aggs results are returned + */ + public void testAggsTimeoutWhileCollecting() { + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setSize(0) + // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set + .setTimeout(new TimeValue(10, TimeUnit.SECONDS)) + .setQuery(new BulkScorerTimeoutQuery(true)) + .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L)); + assertEquals(0, searchResponse.getHits().getHits().length); + StringTerms terms = searchResponse.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + StringTerms.Bucket bucket = terms.getBuckets().get(0); + assertEquals("value", bucket.getKeyAsString()); + assertThat(bucket.getDocCount(), greaterThan(0L)); + }); + } + + /** + * Test the scenario where the suggest phase (part of the query phase) times out, yet there are results + * available coming from executing the query and aggs on each shard. + */ + public void testSuggestTimeoutWithPartialResults() { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("text"); + TimeoutSuggestionBuilder timeoutSuggestionBuilder = new TimeoutSuggestionBuilder(); + suggestBuilder.addSuggestion("suggest", timeoutSuggestionBuilder); + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").suggest(suggestBuilder) + .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L)); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + StringTerms terms = searchResponse.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + StringTerms.Bucket bucket = terms.getBuckets().get(0); + assertEquals("value", bucket.getKeyAsString()); + assertThat(bucket.getDocCount(), greaterThan(0L)); + }); + } + + /** + * Test the scenario where the rescore phase (part of the query phase) times out, yet there are results + * available coming from executing the query and aggs on each shard. + */ + public void testRescoreTimeoutWithPartialResults() { + SearchRequestBuilder searchRequestBuilder = prepareSearch("test").setRescorer(new TimeoutRescorerBuilder()) + .addAggregation(new TermsAggregationBuilder("terms").field("field.keyword")); + ElasticsearchAssertions.assertResponse(searchRequestBuilder, searchResponse -> { + assertThat(searchResponse.isTimedOut(), equalTo(true)); + assertEquals(0, searchResponse.getShardFailures().length); + assertEquals(0, searchResponse.getFailedShards()); + assertThat(searchResponse.getSuccessfulShards(), greaterThan(0)); + assertEquals(searchResponse.getSuccessfulShards(), searchResponse.getTotalShards()); + assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L)); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + StringTerms terms = searchResponse.getAggregations().get("terms"); + assertEquals(1, terms.getBuckets().size()); + StringTerms.Bucket bucket = terms.getBuckets().get(0); + assertEquals("value", bucket.getKeyAsString()); + assertThat(bucket.getDocCount(), greaterThan(0L)); + }); + } + + public void testPartialResultsIntolerantTimeoutBeforeCollecting() { + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test") + // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set + .setTimeout(new TimeValue(10, TimeUnit.SECONDS)) + .setQuery(new BulkScorerTimeoutQuery(false)) + .setAllowPartialSearchResults(false) // this line causes timeouts to report failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(504, ex.status().getStatus()); + } + public void testPartialResultsIntolerantTimeoutWhileCollecting() { ElasticsearchException ex = expectThrows( ElasticsearchException.class, - prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS)) - .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + prepareSearch("test") + // setting the timeout is necessary only because we check that if a TimeExceededException is thrown, a timeout was set + .setTimeout(new TimeValue(10, TimeUnit.SECONDS)) + .setQuery(new BulkScorerTimeoutQuery(true)) .setAllowPartialSearchResults(false) // this line causes timeouts to report failures ); assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(504, ex.status().getStatus()); } - public static class ScriptedTimeoutPlugin extends MockScriptPlugin { - static final String SCRIPT_NAME = "search_timeout"; + public void testPartialResultsIntolerantTimeoutWhileSuggestingOnly() { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("text"); + TimeoutSuggestionBuilder timeoutSuggestionBuilder = new TimeoutSuggestionBuilder(); + suggestBuilder.addSuggestion("suggest", timeoutSuggestionBuilder); + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test").suggest(suggestBuilder).setAllowPartialSearchResults(false) // this line causes timeouts to report + // failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(504, ex.status().getStatus()); + } + + public void testPartialResultsIntolerantTimeoutWhileSuggesting() { + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.setGlobalText("text"); + TimeoutSuggestionBuilder timeoutSuggestionBuilder = new TimeoutSuggestionBuilder(); + suggestBuilder.addSuggestion("suggest", timeoutSuggestionBuilder); + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test").setQuery(new TermQueryBuilder("field", "value")) + .suggest(suggestBuilder) + .setAllowPartialSearchResults(false) // this line causes timeouts to report failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(504, ex.status().getStatus()); + } + + public void testPartialResultsIntolerantTimeoutWhileRescoring() { + ElasticsearchException ex = expectThrows( + ElasticsearchException.class, + prepareSearch("test").setQuery(new TermQueryBuilder("field", "value")) + .setRescorer(new TimeoutRescorerBuilder()) + .setAllowPartialSearchResults(false) // this line causes timeouts to report failures + ); + assertTrue(ex.toString().contains("Time exceeded")); + assertEquals(504, ex.status().getStatus()); + } + + public static final class SearchTimeoutPlugin extends Plugin implements SearchPlugin { + @Override + public List> getQueries() { + return Collections.singletonList(new QuerySpec("timeout", BulkScorerTimeoutQuery::new, parser -> { + throw new UnsupportedOperationException(); + })); + } @Override - public Map, Object>> pluginScripts() { - return Collections.singletonMap(SCRIPT_NAME, params -> { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); + public List> getSuggesters() { + return Collections.singletonList(new SuggesterSpec<>("timeout", TimeoutSuggestionBuilder::new, parser -> { + throw new UnsupportedOperationException(); + }, TermSuggestion::new)); + } + + @Override + public List> getRescorers() { + return Collections.singletonList(new RescorerSpec<>("timeout", TimeoutRescorerBuilder::new, parser -> { + throw new UnsupportedOperationException(); + })); + } + } + + /** + * Query builder that produces a Lucene Query which throws a + * {@link org.elasticsearch.search.internal.ContextIndexSearcher.TimeExceededException} before or while scoring documents. + * This helps make this test not time dependent, otherwise it would be unpredictable when exactly the timeout happens, which is + * rather important if we want to test that we are able to return partial results on timeout. + */ + public static final class BulkScorerTimeoutQuery extends AbstractQueryBuilder { + + private final boolean partialResults; + + BulkScorerTimeoutQuery(boolean partialResults) { + this.partialResults = partialResults; + } + + BulkScorerTimeoutQuery(StreamInput in) throws IOException { + super(in); + this.partialResults = in.readBoolean(); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeBoolean(partialResults); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) {} + + @Override + protected Query doToQuery(SearchExecutionContext context) { + return new Query() { + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { + return new ConstantScoreWeight(this, boost) { + @Override + public BulkScorer bulkScorer(LeafReaderContext context) { + if (partialResults == false) { + ((ContextIndexSearcher) searcher).throwTimeExceededException(); + } + final int maxDoc = context.reader().maxDoc(); + return new BulkScorer() { + @Override + public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException { + max = Math.min(max, maxDoc); + for (int doc = min; doc < max; ++doc) { + if (acceptDocs == null || acceptDocs.get(doc)) { + int docId = doc; + collector.setScorer(new Scorable() { + @Override + public float score() { + return 1f; + } + + @Override + public int docID() { + return docId; + } + }); + collector.collect(doc); + // collect one doc per segment, only then throw a timeout: this ensures partial + // results are returned + ((ContextIndexSearcher) searcher).throwTimeExceededException(); + } + } + // there is a slight chance that no docs are scored for a specific segment. + // other shards / slices will throw the timeout anyway, one is enough. + return max == maxDoc ? DocIdSetIterator.NO_MORE_DOCS : max; + } + + @Override + public long cost() { + return maxDoc; + } + }; + } + + @Override + public Scorer scorer(LeafReaderContext leafReaderContext) { + assert false; + return new ConstantScoreScorer( + this, + score(), + scoreMode, + DocIdSetIterator.all(leafReaderContext.reader().maxDoc()) + ); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; + } + }; + } + + @Override + public String toString(String field) { + return "timeout query"; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override + public boolean equals(Object obj) { + return sameClassAs(obj); + } + + @Override + public int hashCode() { + return classHash(); + } + }; + } + + @Override + protected boolean doEquals(BulkScorerTimeoutQuery other) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + + @Override + public String getWriteableName() { + return "timeout"; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return null; + } + } + + /** + * Suggestion builder that triggers a timeout as part of its execution + */ + private static final class TimeoutSuggestionBuilder extends TermSuggestionBuilder { + TimeoutSuggestionBuilder() { + super("field"); + } + + TimeoutSuggestionBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return "timeout"; + } + + @Override + public SuggestionSearchContext.SuggestionContext build(SearchExecutionContext context) { + return new TimeoutSuggestionContext(new TimeoutSuggester((ContextIndexSearcher) context.searcher()), context); + } + } + + private static final class TimeoutSuggester extends Suggester { + private final ContextIndexSearcher contextIndexSearcher; + + TimeoutSuggester(ContextIndexSearcher contextIndexSearcher) { + this.contextIndexSearcher = contextIndexSearcher; + } + + @Override + protected TermSuggestion innerExecute( + String name, + TimeoutSuggestionContext suggestion, + IndexSearcher searcher, + CharsRefBuilder spare + ) { + contextIndexSearcher.throwTimeExceededException(); + throw new AssertionError("should have thrown TimeExceededException"); + } + + @Override + protected TermSuggestion emptySuggestion(String name, TimeoutSuggestionContext suggestion, CharsRefBuilder spare) { + return new TermSuggestion(name, suggestion.getSize(), SortBy.SCORE); + } + } + + private static final class TimeoutSuggestionContext extends SuggestionSearchContext.SuggestionContext { + TimeoutSuggestionContext(Suggester suggester, SearchExecutionContext searchExecutionContext) { + super(suggester, searchExecutionContext); + } + } + + private static final class TimeoutRescorerBuilder extends RescorerBuilder { + TimeoutRescorerBuilder() { + super(); + } + + TimeoutRescorerBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + protected void doWriteTo(StreamOutput out) {} + + @Override + protected void doXContent(XContentBuilder builder, Params params) {} + + @Override + protected RescoreContext innerBuildContext(int windowSize, SearchExecutionContext context) throws IOException { + return new RescoreContext(10, new Rescorer() { + @Override + public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext rescoreContext) { + ((ContextIndexSearcher) context.searcher()).throwTimeExceededException(); + assert false; + return null; + } + + @Override + public Explanation explain( + int topLevelDocId, + IndexSearcher searcher, + RescoreContext rescoreContext, + Explanation sourceExplanation + ) { + throw new UnsupportedOperationException(); } - return true; }); } + + @Override + public String getWriteableName() { + return "timeout"; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return null; + } + + @Override + public RescorerBuilder rewrite(QueryRewriteContext ctx) { + return this; + } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 8d6ec24efecab..b04ba3e02f3bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -64,6 +65,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.script.MockScriptPlugin.NAME; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.max; @@ -102,7 +104,12 @@ protected Collection> nodePlugins() { public static class CustomScriptPlugin extends MockScriptPlugin { @Override protected Map, Object>> pluginScripts() { - return Collections.singletonMap("5", script -> "5"); + return Map.of("5", script -> "5", "doc['sort'].value", CustomScriptPlugin::sortDoubleScript); + } + + private static Double sortDoubleScript(Map vars) { + Map doc = (Map) vars.get("doc"); + return ((Number) ((ScriptDocValues) doc.get("sort")).get(0)).doubleValue(); } @Override @@ -1268,6 +1275,41 @@ public void testWithRescore() { ); } + public void testScriptSorting() { + Script script = new Script(ScriptType.INLINE, NAME, "doc['sort'].value", Collections.emptyMap()); + assertNoFailuresAndResponse( + prepareSearch("idx").addAggregation( + terms("terms").executionHint(randomExecutionHint()) + .field(TERMS_AGGS_FIELD) + .subAggregation(topHits("hits").sort(SortBuilders.scriptSort(script, ScriptSortType.NUMBER).order(SortOrder.DESC))) + ), + response -> { + Terms terms = response.getAggregations().get("terms"); + assertThat(terms, notNullValue()); + assertThat(terms.getName(), equalTo("terms")); + assertThat(terms.getBuckets().size(), equalTo(5)); + + double higestSortValue = 0; + for (int i = 0; i < 5; i++) { + Terms.Bucket bucket = terms.getBucketByKey("val" + i); + assertThat(bucket, notNullValue()); + assertThat(key(bucket), equalTo("val" + i)); + assertThat(bucket.getDocCount(), equalTo(10L)); + TopHits topHits = bucket.getAggregations().get("hits"); + SearchHits hits = topHits.getHits(); + assertThat(hits.getTotalHits().value, equalTo(10L)); + assertThat(hits.getHits().length, equalTo(3)); + higestSortValue += 10; + assertThat((Double) hits.getAt(0).getSortValues()[0], equalTo(higestSortValue)); + assertThat((Double) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1)); + assertThat((Double) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2)); + + assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5)); + } + } + ); + } + public static class FetchPlugin extends Plugin implements SearchPlugin { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java index d2c7e10f8aa62..9ca5cc3db337a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.search.basic; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -28,6 +29,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +@LuceneTestCase.SuppressFileSystems(value = "HandleLimitFS") // we sometimes have >2048 open files public class SearchWithRandomDisconnectsIT extends AbstractDisruptionTestCase { public void testSearchWithRandomDisconnects() throws InterruptedException, ExecutionException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 0ce4f34463b03..02f743c717231 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -2674,6 +2674,41 @@ public void testPostingsHighlighterOrderByScore() throws Exception { }); } + public void testMaxQueryOffsetDefault() throws Exception { + assertAcked( + prepareCreate("test").setMapping(type1PostingsffsetsMapping()) + .setSettings(Settings.builder().put("index.highlight.max_analyzed_offset", "10").build()) + ); + ensureGreen(); + + prepareIndex("test").setSource( + "field1", + new String[] { + "This sentence contains one match, not that short. This sentence contains zero sentence matches. " + + "This one contains no matches.", + "This is the second value's first sentence. This one contains no matches. " + + "This sentence contains three sentence occurrences (sentence).", + "One sentence match here and scored lower since the text is quite long, not that appealing. " + + "This one contains no matches." } + ).get(); + refresh(); + + // Specific for this test: by passing "-1" as "maxAnalyzedOffset", the index highlight setting above will be used. + SearchSourceBuilder source = searchSource().query(termQuery("field1", "sentence")) + .highlighter(highlight().field("field1").order("score").maxAnalyzedOffset(-1)); + + assertResponse(client().search(new SearchRequest("test").source(source)), response -> { + Map highlightFieldMap = response.getHits().getAt(0).getHighlightFields(); + assertThat(highlightFieldMap.size(), equalTo(1)); + HighlightField field1 = highlightFieldMap.get("field1"); + assertThat(field1.fragments().length, equalTo(1)); + assertThat( + field1.fragments()[0].string(), + equalTo("This sentence contains one match, not that short. This sentence contains zero sentence matches.") + ); + }); + } + public void testPostingsHighlighterEscapeHtml() throws Exception { assertAcked(prepareCreate("test").setMapping("title", "type=text," + randomStoreField() + "index_options=offsets")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index fe49ce57d0400..cbd22856f09a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.Cancellable; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -73,6 +74,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.CancellationException; @@ -591,21 +593,31 @@ public void testNoActiveCopy() throws Exception { private void moveOrCloseShardsOnNodes(String nodeName) throws Exception { final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName); + final ClusterState clusterState = clusterService().state(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { if (randomBoolean()) { closeShardNoCheck(indexShard, randomBoolean()); } else if (randomBoolean()) { final ShardId shardId = indexShard.shardId(); - + final var assignedNodes = new HashSet<>(); + clusterState.routingTable().shardRoutingTable(shardId).allShards().forEach(shr -> { + if (shr.currentNodeId() != null) { + assignedNodes.add(shr.currentNodeId()); + } + if (shr.relocatingNodeId() != null) { + assignedNodes.add(shr.relocatingNodeId()); + } + }); final var targetNodes = new ArrayList(); for (final var targetIndicesService : internalCluster().getInstances(IndicesService.class)) { final var targetNode = targetIndicesService.clusterService().localNode(); - if (targetNode.canContainData() && targetIndicesService.getShardOrNull(shardId) == null) { + if (targetNode.canContainData() + && targetIndicesService.getShardOrNull(shardId) == null + && assignedNodes.contains(targetNode.getId()) == false) { targetNodes.add(targetNode.getId()); } } - if (targetNodes.isEmpty()) { continue; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index c9ab3be0eba61..745120320afab 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -1016,22 +1015,6 @@ public void testRescoreAfterCollapseRandom() throws Exception { }); } - public void testRescoreWithTimeout() throws Exception { - // no dummy docs since merges can change scores while we run queries. - int numDocs = indexRandomNumbers("whitespace", -1, false); - - String intToEnglish = English.intToEnglish(between(0, numDocs - 1)); - String query = intToEnglish.split(" ")[0]; - assertResponse( - prepareSearch().setSearchType(SearchType.QUERY_THEN_FETCH) - .setQuery(QueryBuilders.matchQuery("field1", query).operator(Operator.OR)) - .setSize(10) - .addRescorer(new QueryRescorerBuilder(functionScoreQuery(new TestTimedScoreFunctionBuilder())).windowSize(100)) - .setTimeout(TimeValue.timeValueMillis(10)), - r -> assertTrue(r.isTimedOut()) - ); - } - @Override protected Collection> nodePlugins() { return List.of(TestTimedQueryPlugin.class); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java new file mode 100644 index 0000000000000..c8812cfc109f2 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/RescoreKnnVectorQueryIT.java @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.query; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.VectorIndexType; +import org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.ScriptScoreQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.retriever.KnnRetrieverBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; +import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; +import org.elasticsearch.search.vectors.RescoreVectorBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; +import static org.hamcrest.Matchers.equalTo; + +public class RescoreKnnVectorQueryIT extends ESIntegTestCase { + + public static final String INDEX_NAME = "test"; + public static final String VECTOR_FIELD = "vector"; + public static final String VECTOR_SCORE_SCRIPT = "vector_scoring"; + public static final String QUERY_VECTOR_PARAM = "query_vector"; + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + private static final VectorSimilarityFunction SIMILARITY_FUNCTION = DenseVectorFieldMapper.VectorSimilarity.L2_NORM + .vectorSimilarityFunction(IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT); + + @Override + protected Map, Object>> pluginScripts() { + return Map.of(VECTOR_SCORE_SCRIPT, vars -> { + Map doc = (Map) vars.get("doc"); + return SIMILARITY_FUNCTION.compare( + ((DenseVectorScriptDocValues) doc.get(VECTOR_FIELD)).getVectorValue(), + (float[]) vars.get(QUERY_VECTOR_PARAM) + ); + }); + } + } + + @Before + public void setup() throws IOException { + String type = randomFrom( + Arrays.stream(VectorIndexType.values()) + .filter(VectorIndexType::isQuantized) + .map(t -> t.name().toLowerCase(Locale.ROOT)) + .collect(Collectors.toCollection(ArrayList::new)) + ); + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(VECTOR_FIELD) + .field("type", "dense_vector") + .field("similarity", "l2_norm") + .startObject("index_options") + .field("type", type) + .endObject() + .endObject() + .endObject() + .endObject(); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)) + .build(); + prepareCreate(INDEX_NAME).setMapping(mapping).setSettings(settings).get(); + ensureGreen(INDEX_NAME); + } + + private record TestParams( + int numDocs, + int numDims, + float[] queryVector, + int k, + int numCands, + RescoreVectorBuilder rescoreVectorBuilder + ) { + public static TestParams generate() { + int numDims = randomIntBetween(32, 512) * 2; // Ensure even dimensions + int numDocs = randomIntBetween(10, 100); + int k = randomIntBetween(1, numDocs - 5); + return new TestParams( + numDocs, + numDims, + randomVector(numDims), + k, + (int) (k * randomFloatBetween(1.0f, 10.0f, true)), + new RescoreVectorBuilder(randomFloatBetween(1.0f, 100f, true)) + ); + } + } + + public void testKnnSearchRescore() { + BiFunction knnSearchGenerator = (testParams, requestBuilder) -> { + KnnSearchBuilder knnSearch = new KnnSearchBuilder( + VECTOR_FIELD, + testParams.queryVector, + testParams.k, + testParams.numCands, + testParams.rescoreVectorBuilder, + null + ); + return requestBuilder.setKnnSearch(List.of(knnSearch)); + }; + testKnnRescore(knnSearchGenerator); + } + + public void testKnnQueryRescore() { + BiFunction knnQueryGenerator = (testParams, requestBuilder) -> { + KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder( + VECTOR_FIELD, + testParams.queryVector, + testParams.k, + testParams.numCands, + testParams.rescoreVectorBuilder, + null + ); + return requestBuilder.setQuery(knnQuery); + }; + testKnnRescore(knnQueryGenerator); + } + + public void testKnnRetriever() { + BiFunction knnQueryGenerator = (testParams, requestBuilder) -> { + KnnRetrieverBuilder knnRetriever = new KnnRetrieverBuilder( + VECTOR_FIELD, + testParams.queryVector, + null, + testParams.k, + testParams.numCands, + testParams.rescoreVectorBuilder, + null + ); + return requestBuilder.setSource(new SearchSourceBuilder().retriever(knnRetriever)); + }; + testKnnRescore(knnQueryGenerator); + } + + private void testKnnRescore(BiFunction searchRequestGenerator) { + TestParams testParams = TestParams.generate(); + + int numDocs = testParams.numDocs; + IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; + + for (int i = 0; i < numDocs; i++) { + docs[i] = prepareIndex(INDEX_NAME).setId("" + i).setSource(VECTOR_FIELD, randomVector(testParams.numDims)); + } + indexRandom(true, docs); + + float[] queryVector = testParams.queryVector; + float oversample = randomFloatBetween(1.0f, 100f, true); + RescoreVectorBuilder rescoreVectorBuilder = new RescoreVectorBuilder(oversample); + + SearchRequestBuilder requestBuilder = searchRequestGenerator.apply( + testParams, + prepareSearch(INDEX_NAME).setSize(numDocs).setTrackTotalHits(randomBoolean()) + ); + + assertNoFailuresAndResponse(requestBuilder, knnResponse -> { compareWithExactSearch(knnResponse, queryVector, numDocs); }); + } + + private static void compareWithExactSearch(SearchResponse knnResponse, float[] queryVector, int docCount) { + // Do an exact query and compare + Script script = new Script( + ScriptType.INLINE, + CustomScriptPlugin.NAME, + VECTOR_SCORE_SCRIPT, + Map.of(QUERY_VECTOR_PARAM, queryVector) + ); + ScriptScoreQueryBuilder scriptScoreQueryBuilder = QueryBuilders.scriptScoreQuery(new MatchAllQueryBuilder(), script); + assertNoFailuresAndResponse(prepareSearch(INDEX_NAME).setQuery(scriptScoreQueryBuilder).setSize(docCount), exactResponse -> { + assertHitCount(exactResponse, docCount); + + int i = 0; + SearchHit[] exactHits = exactResponse.getHits().getHits(); + for (SearchHit knnHit : knnResponse.getHits().getHits()) { + while (i < exactHits.length && exactHits[i].getId().equals(knnHit.getId()) == false) { + i++; + } + if (i >= exactHits.length) { + fail("Knn doc not found in exact search"); + } + assertThat("Real score is not the same as rescored score", knnHit.getScore(), equalTo(exactHits[i].getScore())); + } + }); + } + + private static float[] randomVector(int numDimensions) { + float[] vector = new float[numDimensions]; + for (int j = 0; j < numDimensions; j++) { + vector[j] = randomFloatBetween(0, 1, true); + } + return vector; + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 87fb438da973e..ad7dd08654685 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -85,21 +85,21 @@ public static class CustomScriptPlugin extends MockScriptPlugin { @Override protected Map, Object>> pluginScripts() { Map, Object>> scripts = new HashMap<>(); - scripts.put("doc['number'].value", vars -> sortDoubleScript(vars)); - scripts.put("doc['keyword'].value", vars -> sortStringScript(vars)); + scripts.put("doc['number'].value", CustomScriptPlugin::sortDoubleScript); + scripts.put("doc['keyword'].value", CustomScriptPlugin::sortStringScript); return scripts; } - static Double sortDoubleScript(Map vars) { + private static Double sortDoubleScript(Map vars) { Map doc = (Map) vars.get("doc"); - Double index = ((Number) ((ScriptDocValues) doc.get("number")).get(0)).doubleValue(); - return index; + Double score = (Double) vars.get("_score"); + return ((Number) ((ScriptDocValues) doc.get("number")).get(0)).doubleValue() + score; } - static String sortStringScript(Map vars) { + private static String sortStringScript(Map vars) { Map doc = (Map) vars.get("doc"); - String value = ((String) ((ScriptDocValues) doc.get("keyword")).get(0)); - return value; + Double score = (Double) vars.get("_score"); + return ((ScriptDocValues) doc.get("keyword")).get(0) + ",_score=" + score; } } @@ -1763,14 +1763,14 @@ public void testCustomFormat() throws Exception { ); } - public void testScriptFieldSort() throws Exception { + public void testScriptFieldSort() { assertAcked(prepareCreate("test").setMapping("keyword", "type=keyword", "number", "type=integer")); ensureGreen(); final int numDocs = randomIntBetween(10, 20); IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; List keywords = new ArrayList<>(); for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = prepareIndex("test").setSource("number", i, "keyword", Integer.toString(i)); + indexReqs[i] = prepareIndex("test").setSource("number", i, "keyword", Integer.toString(i), "version", i + "." + i); keywords.add(Integer.toString(i)); } Collections.sort(keywords); @@ -1784,7 +1784,7 @@ public void testScriptFieldSort() throws Exception { .addSort(SortBuilders.scriptSort(script, ScriptSortBuilder.ScriptSortType.NUMBER)) .addSort(SortBuilders.scoreSort()), response -> { - double expectedValue = 0; + double expectedValue = 1; // start from 1 because it includes _score, 1.0f for all docs for (SearchHit hit : response.getHits()) { assertThat(hit.getSortValues().length, equalTo(2)); assertThat(hit.getSortValues()[0], equalTo(expectedValue++)); @@ -1805,7 +1805,7 @@ public void testScriptFieldSort() throws Exception { int expectedValue = 0; for (SearchHit hit : response.getHits()) { assertThat(hit.getSortValues().length, equalTo(2)); - assertThat(hit.getSortValues()[0], equalTo(keywords.get(expectedValue++))); + assertThat(hit.getSortValues()[0], equalTo(keywords.get(expectedValue++) + ",_score=1.0")); assertThat(hit.getSortValues()[1], equalTo(1f)); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java index 77c7b4b762e6a..c1549c1f3d384 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java @@ -195,7 +195,7 @@ public void testConcurrentWipeAndRecreateFromOtherCluster() throws IOException { ); assertAcked(clusterAdmin().prepareDeleteRepository(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repoName)); - IOUtils.rm(internalCluster().getCurrentMasterNodeInstance(Environment.class).resolveRepoFile(repoPath.toString())); + IOUtils.rm(internalCluster().getCurrentMasterNodeInstance(Environment.class).resolveRepoDir(repoPath.toString())); createRepository(repoName, "fs", repoPath); createFullSnapshot(repoName, "snap-1"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java index c318ebf78dd96..562f752b82220 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -508,6 +508,12 @@ public void taskSucceeded(ClusterStateTaskListener clusterStateTaskListener, Obj .orElseThrow() .queue(); + // There is one task in the queue for computing and forking the cleanup work. + assertThat(queueLength.getAsInt(), equalTo(1)); + + safeAwait(barrier); // unblock the barrier thread and let it process the queue + safeAwait(barrier); // wait for the queue to be processed + // There are indexCount (=3*snapshotPoolSize) index-deletion tasks, plus one for cleaning up the root metadata. However, the // throttled runner only enqueues one task per SNAPSHOT thread to start with, and then the eager runner adds another one. This shows // we are not spamming the threadpool with all the tasks at once, which means that other snapshot activities can run alongside this diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java index 53263468bf0de..75cd9240c319f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java @@ -72,6 +72,7 @@ public void testRestartNodeDuringSnapshot() throws Exception { final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); final var snapshotFuture = startFullSnapshotBlockedOnDataNode(randomIdentifier(), repoName, originalNode); + safeAwait((ActionListener l) -> flushMasterQueue(clusterService, l)); final var snapshotCompletesWithoutPausingListener = ClusterServiceUtils.addTemporaryStateListener(clusterService, state -> { final var entriesForRepo = SnapshotsInProgress.get(state).forRepo(repoName); if (entriesForRepo.isEmpty()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java index b9e47740e2945..b86cae1c2fb60 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -17,10 +17,12 @@ import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.List; import java.util.concurrent.TimeUnit; @@ -223,4 +225,30 @@ public void testRerouteWhenShardSnapshotsCompleted() throws Exception { safeAwait(shardMovedListener); ensureGreen(indexName); } + + @TestLogging(reason = "testing task description, logged at DEBUG", value = "org.elasticsearch.cluster.service.MasterService:DEBUG") + public void testCreateSnapshotTaskDescription() { + createIndexWithRandomDocs(randomIdentifier(), randomIntBetween(1, 5)); + final var repositoryName = randomIdentifier(); + createRepository(repositoryName, "mock"); + + final var snapshotName = randomIdentifier(); + MockLog.assertThatLogger( + () -> createFullSnapshot(repositoryName, snapshotName), + MasterService.class, + new MockLog.SeenEventExpectation( + "executing cluster state update debug message", + MasterService.class.getCanonicalName(), + Level.DEBUG, + "executing cluster state update for [create_snapshot [" + + snapshotName + + "][CreateSnapshotTask{repository=" + + repositoryName + + ", snapshot=*" + + snapshotName + + "*}]]" + ) + ); + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java similarity index 62% rename from server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java rename to server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java index c6e02300ccef6..27fb2c46d4d49 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java @@ -10,12 +10,21 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; import org.elasticsearch.indices.AssociatedIndexDescriptor; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptorUtils; import org.elasticsearch.plugins.Plugin; @@ -24,10 +33,12 @@ import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -41,9 +52,10 @@ import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.oneOf; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) -public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { +public class SystemResourceSnapshotIT extends AbstractSnapshotIntegTestCase { public static final String REPO_NAME = "test-repo"; @@ -55,6 +67,11 @@ protected Collection> nodePlugins() { plugins.add(SystemIndexTestPlugin.class); plugins.add(AnotherSystemIndexTestPlugin.class); plugins.add(AssociatedIndicesTestPlugin.class); + plugins.add(DataStreamsPlugin.class); + plugins.add(AnotherSystemDataStreamTestPlugin.class); + plugins.add(SystemDataStreamTestPlugin.class); + plugins.add(SystemDataStreamManyShardsTestPlugin.class); + plugins.add(AssociatedIndicesSystemDSTestPlugin.class); return plugins; } @@ -70,16 +87,18 @@ public void setup() { */ public void testRestoreSystemIndicesAsGlobalState() { createRepository(REPO_NAME, "fs"); - // put a document in a system index + // put a document in a system index and data stream indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); // run a snapshot including global state createFullSnapshot(REPO_NAME, "test-snap"); - // add another document + // add another document to each system resource indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); @@ -91,8 +110,9 @@ public void testRestoreSystemIndicesAsGlobalState() { ).setWaitForCompletion(true).setRestoreGlobalState(true).get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - // verify only the original document is restored + // verify only the original documents are restored assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); } /** @@ -101,6 +121,7 @@ public void testRestoreSystemIndicesAsGlobalState() { public void testSnapshotWithoutGlobalState() { createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "system index doc"); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc("not-a-system-index", "1", "purpose", "non system index doc"); // run a snapshot without global state @@ -122,6 +143,7 @@ public void testSnapshotWithoutGlobalState() { assertThat("not-a-system-index", in(snapshottedIndices)); assertThat(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, not(in(snapshottedIndices))); + assertThat(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, not(in(snapshottedIndices))); } /** @@ -131,23 +153,44 @@ public void testSnapshotByFeature() { createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + refresh( + SystemIndexTestPlugin.SYSTEM_INDEX_NAME, + AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, + SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, + AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME + ); // snapshot by feature CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") .setIncludeGlobalState(true) .setWaitForCompletion(true) - .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName(), AnotherSystemIndexTestPlugin.class.getSimpleName()) + .setFeatureStates( + SystemIndexTestPlugin.class.getSimpleName(), + AnotherSystemIndexTestPlugin.class.getSimpleName(), + SystemDataStreamTestPlugin.class.getSimpleName(), + AnotherSystemDataStreamTestPlugin.class.getSimpleName() + ) .get(); assertSnapshotSuccess(createSnapshotResponse); // add some other documents indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + refresh( + SystemIndexTestPlugin.SYSTEM_INDEX_NAME, + AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, + SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, + AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME + ); assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); // restore indices as global state without closing the index RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( @@ -160,6 +203,8 @@ public void testSnapshotByFeature() { // verify only the original document is restored assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); + assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); } /** @@ -175,7 +220,8 @@ public void testDefaultRestoreOnlyRegularIndices() { indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); // snapshot including global state CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") @@ -193,7 +239,11 @@ public void testDefaultRestoreOnlyRegularIndices() { assertThat(restoreResponse.getRestoreInfo().totalShards(), greaterThan(0)); assertThat( restoreResponse.getRestoreInfo().indices(), - allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME))) + allOf( + hasItem(regularIndex), + not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)), + not(hasItem(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME)) + ) ); } @@ -207,7 +257,15 @@ public void testRestoreByFeature() { indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + refresh( + regularIndex, + SystemIndexTestPlugin.SYSTEM_INDEX_NAME, + AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, + SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, + AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME + ); // snapshot including global state CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") @@ -219,10 +277,19 @@ public void testRestoreByFeature() { // add some other documents indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); - + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + refresh( + regularIndex, + SystemIndexTestPlugin.SYSTEM_INDEX_NAME, + AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, + SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, + AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME + ); assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); // Delete the regular index so we can restore it assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); @@ -232,14 +299,16 @@ public void testRestoreByFeature() { TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap" - ).setWaitForCompletion(true).setFeatureStates("SystemIndexTestPlugin").get(); + ).setWaitForCompletion(true).setFeatureStates("SystemIndexTestPlugin", "SystemDataStreamTestPlugin").get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - // verify that the restored system index has only one document + // verify that the restored system index and data stream each only have one document assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); - // but the non-requested feature should still have its new document + // but the non-requested features should still have their new documents assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); } /** @@ -254,36 +323,58 @@ public void testSnapshotAndRestoreAssociatedIndices() { indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME); + indexDataStream(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + + refresh( + regularIndex, + AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, + AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, + AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME, + AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME + ); // snapshot CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") - .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) + .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName(), AssociatedIndicesSystemDSTestPlugin.class.getSimpleName()) .setWaitForCompletion(true) .get(); assertSnapshotSuccess(createSnapshotResponse); // verify the correctness of the snapshot - Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, REPO_NAME) - .get() - .getSnapshots() + var snapshotsResponse = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, REPO_NAME).get(); + Set snapshottedIndices = snapshotsResponse.getSnapshots() .stream() .map(SnapshotInfo::indices) .flatMap(Collection::stream) .collect(Collectors.toSet()); + Set snapshottedDataStreams = snapshotsResponse.getSnapshots() + .stream() + .map(SnapshotInfo::dataStreams) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME)); assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME)); + assertThat(snapshottedDataStreams, hasItem(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME)); + assertThat(snapshottedIndices, hasItem(AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME)); // add some other documents indexDoc(regularIndex, "2", "purpose", "post-snapshot doc"); indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); - refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME); assertThat(getDocCount(regularIndex), equalTo(2L)); assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); // And delete the associated index so we can restore it - assertAcked(indicesAdmin().prepareDelete(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME).get()); + assertAcked( + indicesAdmin().prepareDelete( + AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, + AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME + ).get() + ); // restore the feature state and its associated index RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( @@ -291,15 +382,17 @@ public void testSnapshotAndRestoreAssociatedIndices() { REPO_NAME, "test-snap" ) - .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME) + .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME) .setWaitForCompletion(true) - .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) + .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName(), AssociatedIndicesSystemDSTestPlugin.class.getSimpleName()) .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); // verify only the original document is restored assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); assertThat(getDocCount(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); + assertThat(getDocCount(AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L)); } /** @@ -308,7 +401,8 @@ public void testSnapshotAndRestoreAssociatedIndices() { public void testRestoreFeatureNotInSnapshot() { createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); // snapshot including global state CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") @@ -322,7 +416,7 @@ public void testRestoreFeatureNotInSnapshot() { SnapshotRestoreException.class, clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") .setWaitForCompletion(true) - .setFeatureStates("SystemIndexTestPlugin", fakeFeatureStateName) + .setFeatureStates("SystemIndexTestPlugin", "SystemDataStreamTestPlugin", fakeFeatureStateName) ); assertThat( @@ -438,7 +532,8 @@ public void testSystemIndicesCannotBeRenamed() { public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); // run a snapshot including global state CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") @@ -449,9 +544,11 @@ public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { // add another document indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); // restore indices as global state a null list of feature states RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( @@ -463,6 +560,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { // verify that the system index is destroyed assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); } /** @@ -473,8 +571,9 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() { createRepository(REPO_NAME, "fs"); String regularIndex = "my-index"; indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, regularIndex); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, regularIndex); // run a snapshot including global state CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") @@ -485,10 +584,12 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() { // add another document indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME); assertAcked(indicesAdmin().prepareDelete(regularIndex).get()); assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); // restore with global state and all indices but explicitly no feature states. RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( @@ -500,6 +601,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() { // verify that the system index still has the updated document, i.e. has not been restored assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L)); // And the regular index has been restored assertThat(getDocCount(regularIndex), equalTo(1L)); } @@ -564,6 +666,8 @@ public void testSystemIndexAliasesAreAlwaysRestored() { // Create a system index final String systemIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; indexDoc(systemIndexName, "1", "purpose", "pre-snapshot doc"); + // Create a system data stream + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); // And a regular index // And a regular index so we can avoid matching all indices on the restore @@ -603,6 +707,109 @@ public void testSystemIndexAliasesAreAlwaysRestored() { } + public void testSystemDataStreamAliasesAreAlwaysRestored() { + createRepository(REPO_NAME, "fs"); + // Create a system data stream + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + + // And a regular index so we can avoid matching all indices on the restore + final String regularIndex = "regular-index"; + final String regularAlias = "regular-alias"; + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + + // And make sure they both have aliases + final String systemDataStreamAlias = SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME + "-alias"; + assertAcked( + indicesAdmin().prepareAliases() + .addAlias(regularIndex, regularAlias) + .addAlias(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, systemDataStreamAlias, true) + .get() + ); + + // And add a doc to ensure the alias works + indexDataStream(systemDataStreamAlias, "2", "purpose", "post-alias doc"); + + // Run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // And delete the regular index and system data stream + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + assertAcked( + client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME) + ).actionGet() + ); + + // Now restore the snapshot with no aliases + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( + TEST_REQUEST_TIMEOUT, + REPO_NAME, + "test-snap" + ) + .setFeatureStates("SystemDataStreamTestPlugin") + .setWaitForCompletion(true) + .setRestoreGlobalState(false) + .setIncludeAliases(false) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The regular index should exist + assertTrue(indexExists(regularIndex)); + assertFalse(indexExists(regularAlias)); + + // And the system data stream, queried by alias, should have 2 docs + assertTrue(indexExists(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME)); + assertTrue(indexExists(systemDataStreamAlias)); + assertThat(getDocCount(systemDataStreamAlias), equalTo(2L)); + } + + public void testDeletedDatastreamIsRestorable() { + createRepository(REPO_NAME, "fs"); + // Create a system data stream + indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc"); + + // And a regular index so we can avoid matching all indices on the restore + final String regularIndex = "regular-index"; + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + + // Run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // And delete the regular index and system data stream + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + assertAcked( + client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME) + ).actionGet() + ); + + // Now restore the snapshot with no aliases + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( + TEST_REQUEST_TIMEOUT, + REPO_NAME, + "test-snap" + ) + .setFeatureStates("SystemDataStreamTestPlugin") + .setWaitForCompletion(true) + .setRestoreGlobalState(false) + .setIncludeAliases(false) + .get(); + + // And the system data stream, queried by alias, should have 2 docs + assertTrue(indexExists(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME)); + assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L)); + } + /** * Tests that the special "none" feature state name cannot be combined with other * feature state names, and an error occurs if it's tried. @@ -739,6 +946,61 @@ public void testPartialSnapshotsOfSystemIndexRemovesFeatureState() throws Except }); } + /** + * Ensures that if we can only capture a partial snapshot of a system data stream, then the feature state associated + * with that data stream is not included in the snapshot, because it would not be safe to restore that feature state. + */ + @AwaitsFix(bugUrl = "ES-11251") + public void testPartialSnapshotsOfSystemDataStreamRemovesFeatureState() throws Exception { + final String partialIndexName = SystemDataStreamManyShardsTestPlugin.SYSTEM_DATASTREAM_NAME; + final String fullIndexName = AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME; + + createRepositoryNoVerify(REPO_NAME, "mock"); + + // Create the index that we'll get a partial snapshot of with a bunch of shards + indexDataStream(partialIndexName, "1", "purpose", "pre-snapshot doc"); + // And another one with the default + indexDataStream(fullIndexName, "1", "purpose", "pre-snapshot doc"); + ensureGreen(); + + // Stop a random data node so we lose a shard from the partial index + internalCluster().stopRandomDataNode(); + assertBusy(() -> { + var status = clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).get().getStatus(); + assertThat(status, oneOf(ClusterHealthStatus.YELLOW, ClusterHealthStatus.RED)); + }, 30, TimeUnit.SECONDS); + + // Get ready to block + blockMasterFromFinalizingSnapshotOnIndexFile(REPO_NAME); + + // Start a snapshot and wait for it to hit the block, then kill the master to force a failover + final String partialSnapName = "test-partial-snap"; + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot( + TEST_REQUEST_TIMEOUT, + REPO_NAME, + partialSnapName + ).setIncludeGlobalState(true).setWaitForCompletion(false).setPartial(true).get(); + assertThat(createSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + waitForBlock(internalCluster().getMasterName(), REPO_NAME); + internalCluster().stopCurrentMasterNode(); + + // Now get the snapshot and do our checks + assertBusy(() -> { + GetSnapshotsResponse snapshotsStatusResponse = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, REPO_NAME) + .setSnapshots(partialSnapName) + .get(); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + assertNotNull(snapshotInfo); + assertThat(snapshotInfo.failedShards(), lessThan(snapshotInfo.totalShards())); + List statesInSnapshot = snapshotInfo.featureStates().stream().map(SnapshotFeatureInfo::getPluginName).toList(); + assertThat(statesInSnapshot, not(hasItem((new SystemDataStreamManyShardsTestPlugin()).getFeatureName()))); + assertThat(statesInSnapshot, hasItem((new AnotherSystemDataStreamTestPlugin()).getFeatureName())); + }, 5L, TimeUnit.SECONDS); + + // Cleanup to prevent unrelated shutdown failures + internalCluster().startDataOnlyNode(); + } + public void testParallelIndexDeleteRemovesFeatureState() throws Exception { final String indexToBeDeleted = SystemIndexTestPlugin.SYSTEM_INDEX_NAME; final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME; @@ -814,6 +1076,14 @@ private long getDocCount(String indexName) { return indicesAdmin().prepareStats(indexName).get().getPrimaries().getDocs().getCount(); } + private DocWriteResponse indexDataStream(String index, String id, String... source) { + var sourceWithTimestamp = new String[source.length + 2]; + sourceWithTimestamp[0] = "@timestamp"; + sourceWithTimestamp[1] = Long.toString(System.currentTimeMillis()); + System.arraycopy(source, 0, sourceWithTimestamp, 2, source.length); + return prepareIndex(index).setId(id).setSource((Object[]) sourceWithTimestamp).setOpType(DocWriteRequest.OpType.CREATE).get(); + } + public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { public static final String SYSTEM_INDEX_NAME = ".test-system-idx"; @@ -858,6 +1128,126 @@ public String getFeatureDescription() { } } + public static class SystemDataStreamTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_DATASTREAM_NAME = ".test-system-data-stream"; + + @Override + public Collection getSystemDataStreamDescriptors() { + try { + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}"); + return Collections.singletonList( + new SystemDataStreamDescriptor( + SYSTEM_DATASTREAM_NAME, + "system data stream test", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct? + .template(new Template(Settings.EMPTY, mappings, null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(), + Map.of(), + List.of("product"), + "product", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getFeatureName() { + return SystemDataStreamTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "A simple test plugin for data streams"; + } + } + + public static class SystemDataStreamManyShardsTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_DATASTREAM_NAME = ".test-system-data-stream-many-shards"; + + @Override + public Collection getSystemDataStreamDescriptors() { + try { + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}"); + return Collections.singletonList( + new SystemDataStreamDescriptor( + SYSTEM_DATASTREAM_NAME, + "system data stream test", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct? + .template(new Template(indexSettings(6, 0).build(), mappings, null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(), + Map.of(), + List.of("product"), + "product", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getFeatureName() { + return SystemDataStreamManyShardsTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "A simple test plugin for data streams"; + } + } + + public static class AnotherSystemDataStreamTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_DATASTREAM_NAME = ".another-test-system-data-stream"; + + @Override + public Collection getSystemDataStreamDescriptors() { + try { + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}"); + return Collections.singletonList( + new SystemDataStreamDescriptor( + SYSTEM_DATASTREAM_NAME, + "another system data stream test", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct? + .template(new Template(Settings.EMPTY, mappings, null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(), + Map.of(), + List.of("product"), + "product", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getFeatureName() { + return AnotherSystemDataStreamTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "Another simple test plugin for data streams"; + } + } + public static class AssociatedIndicesTestPlugin extends Plugin implements SystemIndexPlugin { public static final String SYSTEM_INDEX_NAME = ".third-test-system-idx"; @@ -885,4 +1275,50 @@ public String getFeatureDescription() { return "Another simple test plugin"; } } + + public static class AssociatedIndicesSystemDSTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_DATASTREAM_NAME = ".test-system-data-stream-two"; + public static final String ASSOCIATED_INDEX_NAME = ".associated-idx2"; + + @Override + public Collection getSystemDataStreamDescriptors() { + try { + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}"); + return Collections.singletonList( + new SystemDataStreamDescriptor( + SYSTEM_DATASTREAM_NAME, + "system data stream test", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct? + .template(new Template(Settings.EMPTY, mappings, null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(), + Map.of(), + List.of("product"), + "product", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Collection getAssociatedIndexDescriptors() { + return Collections.singletonList(new AssociatedIndexDescriptor(ASSOCIATED_INDEX_NAME, "Associated indices")); + } + + @Override + public String getFeatureName() { + return AssociatedIndicesSystemDSTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "Another simple test plugin"; + } + } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 8599439d7e75a..a6961e3e5fa28 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -62,7 +62,6 @@ exports org.elasticsearch.action.admin.cluster.coordination; exports org.elasticsearch.action.admin.cluster.desirednodes; exports org.elasticsearch.action.admin.cluster.health; - exports org.elasticsearch.action.admin.cluster.migration; exports org.elasticsearch.action.admin.cluster.node.capabilities; exports org.elasticsearch.action.admin.cluster.node.hotthreads; exports org.elasticsearch.action.admin.cluster.node.info; @@ -282,6 +281,7 @@ exports org.elasticsearch.indices.recovery; exports org.elasticsearch.indices.recovery.plan; exports org.elasticsearch.indices.store; + exports org.elasticsearch.indices.system; exports org.elasticsearch.inference; exports org.elasticsearch.ingest; exports org.elasticsearch.internal @@ -380,7 +380,6 @@ exports org.elasticsearch.tasks; exports org.elasticsearch.threadpool; exports org.elasticsearch.transport; - exports org.elasticsearch.upgrades; exports org.elasticsearch.usage; exports org.elasticsearch.watcher; diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index 37f6e6434735c..7437985e49636 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -98,6 +98,32 @@ public static TransportVersion current() { return CurrentHolder.CURRENT; } + /** + * @return whether this is a known {@link TransportVersion}, i.e. one declared in {@link TransportVersions} or which dates back to + * before 8.9.0 when they matched the release versions exactly and there was no branching or patching. Other versions may exist + * in the wild (they're sent over the wire by numeric ID) but we don't know how to communicate using such versions. + */ + public boolean isKnown() { + return before(TransportVersions.V_8_9_X) || TransportVersions.VERSION_IDS.containsKey(id); + } + + /** + * @return the newest known {@link TransportVersion} which is no older than this instance. Returns {@link TransportVersions#ZERO} if + * there are no such versions. + */ + public TransportVersion bestKnownVersion() { + if (isKnown()) { + return this; + } + TransportVersion bestSoFar = TransportVersions.ZERO; + for (final var knownVersion : TransportVersions.VERSION_IDS.values()) { + if (knownVersion.after(bestSoFar) && knownVersion.before(this)) { + bestSoFar = knownVersion; + } + } + return bestSoFar; + } + public static TransportVersion fromString(String str) { return TransportVersion.fromId(Integer.parseInt(str)); } @@ -110,20 +136,20 @@ public static TransportVersion fromString(String str) { * When a patch version of an existing transport version is created, {@code transportVersion.isPatchFrom(patchVersion)} * will match any transport version at or above {@code patchVersion} that is also of the same base version. *

- * For example, {@code version.isPatchFrom(8_800_00_4)} will return the following for the given {@code version}: + * For example, {@code version.isPatchFrom(8_800_0_04)} will return the following for the given {@code version}: *

    - *
  • {@code 8_799_00_0.isPatchFrom(8_800_00_4)}: {@code false}
    • - *
  • {@code 8_799_00_9.isPatchFrom(8_800_00_4)}: {@code false}
    • - *
  • {@code 8_800_00_0.isPatchFrom(8_800_00_4)}: {@code false}
    • - *
  • {@code 8_800_00_3.isPatchFrom(8_800_00_4)}: {@code false}
    • - *
  • {@code 8_800_00_4.isPatchFrom(8_800_00_4)}: {@code true}
    • - *
  • {@code 8_800_00_9.isPatchFrom(8_800_00_4)}: {@code true}
    • - *
  • {@code 8_800_01_0.isPatchFrom(8_800_00_4)}: {@code false}
    • - *
  • {@code 8_801_00_0.isPatchFrom(8_800_00_4)}: {@code false}
    • + *
  • {@code 8_799_0_00.isPatchFrom(8_800_0_04)}: {@code false}
    • + *
  • {@code 8_799_0_09.isPatchFrom(8_800_0_04)}: {@code false}
    • + *
  • {@code 8_800_0_00.isPatchFrom(8_800_0_04)}: {@code false}
    • + *
  • {@code 8_800_0_03.isPatchFrom(8_800_0_04)}: {@code false}
    • + *
  • {@code 8_800_0_04.isPatchFrom(8_800_0_04)}: {@code true}
    • + *
  • {@code 8_800_0_49.isPatchFrom(8_800_0_04)}: {@code true}
    • + *
  • {@code 8_800_1_00.isPatchFrom(8_800_0_04)}: {@code false}
    • + *
  • {@code 8_801_0_00.isPatchFrom(8_800_0_04)}: {@code false}
    • *
*/ public boolean isPatchFrom(TransportVersion version) { - return onOrAfter(version) && id < version.id + 10 - (version.id % 10); + return onOrAfter(version) && id < version.id + 100 - (version.id % 100); } /** diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 603fc6a32f078..83d22f31b1c80 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -96,87 +96,96 @@ static TransportVersion def(int id) { */ public static final TransportVersion V_8_9_X = def(8_500_020); public static final TransportVersion V_8_10_X = def(8_500_061); - public static final TransportVersion V_8_11_X = def(8_512_00_1); - public static final TransportVersion V_8_12_0 = def(8_560_00_0); - public static final TransportVersion V_8_12_1 = def(8_560_00_1); - public static final TransportVersion V_8_13_0 = def(8_595_00_0); - public static final TransportVersion V_8_13_4 = def(8_595_00_1); - public static final TransportVersion V_8_14_0 = def(8_636_00_1); - public static final TransportVersion V_8_15_0 = def(8_702_00_2); - public static final TransportVersion V_8_15_2 = def(8_702_00_3); - public static final TransportVersion V_8_16_0 = def(8_772_00_1); - public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16 = def(8_772_00_2); - public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE_BACKPORT_8_16 = def(8_772_00_3); - public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16 = def(8_772_00_4); - public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0); - public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0); - public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0); - public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_00_0); - public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_00_0); - public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_00_0); - public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_00_0); - public static final TransportVersion INDEX_REQUEST_REMOVE_METERING = def(8_780_00_0); - public static final TransportVersion CPU_STAT_STRING_PARSING = def(8_781_00_0); - public static final TransportVersion QUERY_RULES_RETRIEVER = def(8_782_00_0); - public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0); - public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0); - public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0); - public static final TransportVersion KQL_QUERY_ADDED = def(8_786_00_0); - public static final TransportVersion ROLE_MONITOR_STATS = def(8_787_00_0); - public static final TransportVersion DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK = def(8_788_00_0); - public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_789_00_0); - public static final TransportVersion VERTEX_AI_INPUT_TYPE_ADDED = def(8_790_00_0); - public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE = def(8_791_00_0); - public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES = def(8_792_00_0); - public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS = def(8_793_00_0); - public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS_REVERT = def(8_794_00_0); - public static final TransportVersion FAST_REFRESH_RCO_2 = def(8_795_00_0); - public static final TransportVersion ESQL_ENRICH_RUNTIME_WARNINGS = def(8_796_00_0); - public static final TransportVersion INGEST_PIPELINE_CONFIGURATION_AS_MAP = def(8_797_00_0); - public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE_FIX_8_17 = def(8_797_00_1); - public static final TransportVersion SOURCE_MODE_TELEMETRY_FIX_8_17 = def(8_797_00_2); - public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_00_0); - public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_00_0); - public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_00_0); - public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE = def(8_801_00_0); - public static final TransportVersion SOURCE_MODE_TELEMETRY = def(8_802_00_0); - public static final TransportVersion NEW_REFRESH_CLUSTER_BLOCK = def(8_803_00_0); - public static final TransportVersion RETRIES_AND_OPERATIONS_IN_BLOBSTORE_STATS = def(8_804_00_0); - public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_00_0); - public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_00_0); - public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_00_0); - public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_00_0); - public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_809_00_0); - public static final TransportVersion NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION = def(8_810_00_0); - public static final TransportVersion ERROR_TRACE_IN_TRANSPORT_HEADER = def(8_811_00_0); - public static final TransportVersion FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING = def(8_812_00_0); - public static final TransportVersion SIMULATE_IGNORED_FIELDS = def(8_813_00_0); - public static final TransportVersion TRANSFORMS_UPGRADE_MODE = def(8_814_00_0); - public static final TransportVersion NODE_SHUTDOWN_EPHEMERAL_ID_ADDED = def(8_815_00_0); - public static final TransportVersion ESQL_CCS_TELEMETRY_STATS = def(8_816_00_0); - public static final TransportVersion TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID = def(8_817_00_0); - public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0); - public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_00_0); - public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0); - public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_00_0); - public static final TransportVersion ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION = def(8_822_00_0); - public static final TransportVersion KQL_QUERY_TECH_PREVIEW = def(8_823_00_0); - public static final TransportVersion ESQL_PROFILE_ROWS_PROCESSED = def(8_824_00_0); - public static final TransportVersion BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1 = def(8_825_00_0); - public static final TransportVersion REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1 = def(8_826_00_0); - public static final TransportVersion ESQL_SKIP_ES_INDEX_SERIALIZATION = def(8_827_00_0); - public static final TransportVersion ADD_INDEX_BLOCK_TWO_PHASE = def(8_828_00_0); - public static final TransportVersion RESOLVE_CLUSTER_NO_INDEX_EXPRESSION = def(8_829_00_0); - public static final TransportVersion ML_ROLLOVER_LEGACY_INDICES = def(8_830_00_0); - public static final TransportVersion ADD_INCLUDE_FAILURE_INDICES_OPTION = def(8_831_00_0); - public static final TransportVersion ESQL_RESPONSE_PARTIAL = def(8_832_00_0); - public static final TransportVersion RANK_DOC_OPTIONAL_METADATA_FOR_EXPLAIN = def(8_833_00_0); - public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_ADD_REPLICATE_FOR = def(8_834_00_0); - public static final TransportVersion INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR = def(8_835_00_0); - public static final TransportVersion RESOURCE_DEPRECATION_CHECKS = def(8_836_00_0); - public static final TransportVersion LINEAR_RETRIEVER_SUPPORT = def(8_837_00_0); - public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_00_0); - public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_00_0); + public static final TransportVersion V_8_11_X = def(8_512_0_01); + public static final TransportVersion V_8_12_0 = def(8_560_0_00); + public static final TransportVersion V_8_12_1 = def(8_560_0_01); + public static final TransportVersion V_8_13_0 = def(8_595_0_00); + public static final TransportVersion V_8_13_4 = def(8_595_0_01); + public static final TransportVersion V_8_14_0 = def(8_636_0_01); + public static final TransportVersion V_8_15_0 = def(8_702_0_02); + public static final TransportVersion V_8_15_2 = def(8_702_0_03); + public static final TransportVersion V_8_16_0 = def(8_772_0_01); + public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16 = def(8_772_0_02); + public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE_BACKPORT_8_16 = def(8_772_0_03); + public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16 = def(8_772_0_04); + public static final TransportVersion INITIAL_ELASTICSEARCH_8_16_5 = def(8_772_0_05); + public static final TransportVersion INITIAL_ELASTICSEARCH_8_16_6 = def(8_772_0_06); + public static final TransportVersion INITIAL_ELASTICSEARCH_8_16_7 = def(8_772_0_07); + public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_0_00); + public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_0_00); + public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_0_00); + public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_0_00); + public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_0_00); + public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_0_00); + public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_0_00); + public static final TransportVersion INDEX_REQUEST_REMOVE_METERING = def(8_780_0_00); + public static final TransportVersion CPU_STAT_STRING_PARSING = def(8_781_0_00); + public static final TransportVersion QUERY_RULES_RETRIEVER = def(8_782_0_00); + public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_0_00); + public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_0_00); + public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_0_00); + public static final TransportVersion KQL_QUERY_ADDED = def(8_786_0_00); + public static final TransportVersion ROLE_MONITOR_STATS = def(8_787_0_00); + public static final TransportVersion DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK = def(8_788_0_00); + public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_789_0_00); + public static final TransportVersion VERTEX_AI_INPUT_TYPE_ADDED = def(8_790_0_00); + public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE = def(8_791_0_00); + public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES = def(8_792_0_00); + public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS = def(8_793_0_00); + public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS_REVERT = def(8_794_0_00); + public static final TransportVersion FAST_REFRESH_RCO_2 = def(8_795_0_00); + public static final TransportVersion ESQL_ENRICH_RUNTIME_WARNINGS = def(8_796_0_00); + public static final TransportVersion INGEST_PIPELINE_CONFIGURATION_AS_MAP = def(8_797_0_00); + public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE_FIX_8_17 = def(8_797_0_01); + public static final TransportVersion SOURCE_MODE_TELEMETRY_FIX_8_17 = def(8_797_0_02); + public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_3 = def(8_797_0_03); + public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_4 = def(8_797_0_04); + public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_5 = def(8_797_0_05); + public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_0_00); + public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_0_00); + public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_0_00); + public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE = def(8_801_0_00); + public static final TransportVersion SOURCE_MODE_TELEMETRY = def(8_802_0_00); + public static final TransportVersion NEW_REFRESH_CLUSTER_BLOCK = def(8_803_0_00); + public static final TransportVersion RETRIES_AND_OPERATIONS_IN_BLOBSTORE_STATS = def(8_804_0_00); + public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_0_00); + public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_0_00); + public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_0_00); + public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_0_00); + public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_809_0_00); + public static final TransportVersion NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION = def(8_810_0_00); + public static final TransportVersion ERROR_TRACE_IN_TRANSPORT_HEADER = def(8_811_0_00); + public static final TransportVersion FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING = def(8_812_0_00); + public static final TransportVersion SIMULATE_IGNORED_FIELDS = def(8_813_0_00); + public static final TransportVersion TRANSFORMS_UPGRADE_MODE = def(8_814_0_00); + public static final TransportVersion NODE_SHUTDOWN_EPHEMERAL_ID_ADDED = def(8_815_0_00); + public static final TransportVersion ESQL_CCS_TELEMETRY_STATS = def(8_816_0_00); + public static final TransportVersion TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID = def(8_817_0_00); + public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_0_00); + public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_0_00); + public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_0_00); + public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_0_00); + public static final TransportVersion ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION = def(8_822_0_00); + public static final TransportVersion KQL_QUERY_TECH_PREVIEW = def(8_823_0_00); + public static final TransportVersion ESQL_PROFILE_ROWS_PROCESSED = def(8_824_0_00); + public static final TransportVersion BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1 = def(8_825_0_00); + public static final TransportVersion REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1 = def(8_826_0_00); + public static final TransportVersion ESQL_SKIP_ES_INDEX_SERIALIZATION = def(8_827_0_00); + public static final TransportVersion ADD_INDEX_BLOCK_TWO_PHASE = def(8_828_0_00); + public static final TransportVersion RESOLVE_CLUSTER_NO_INDEX_EXPRESSION = def(8_829_0_00); + public static final TransportVersion ML_ROLLOVER_LEGACY_INDICES = def(8_830_0_00); + public static final TransportVersion ADD_INCLUDE_FAILURE_INDICES_OPTION = def(8_831_0_00); + public static final TransportVersion ESQL_RESPONSE_PARTIAL = def(8_832_0_00); + public static final TransportVersion RANK_DOC_OPTIONAL_METADATA_FOR_EXPLAIN = def(8_833_0_00); + public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_ADD_REPLICATE_FOR = def(8_834_0_00); + public static final TransportVersion INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR = def(8_835_0_00); + public static final TransportVersion RESOURCE_DEPRECATION_CHECKS = def(8_836_0_00); + public static final TransportVersion LINEAR_RETRIEVER_SUPPORT = def(8_837_0_00); + public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_0_00); + public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_0_00); + public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_00_0); + public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_18 = def(8_840_00_1); + public static final TransportVersion RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR_8_18 = def(8_840_0_02); /* * STOP! READ THIS FIRST! No, really, @@ -193,17 +202,17 @@ static TransportVersion def(int id) { * To add a new transport version, add a new constant at the bottom of the list, above this comment. Don't add other lines, * comments, etc. The version id has the following layout: * - * M_NNN_SS_P + * M_NNN_S_PP * * M - The major version of Elasticsearch * NNN - The server version part - * SS - The serverless version part. It should always be 00 here, it is used by serverless only. - * P - The patch version part + * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. + * PP - The patch version part * * To determine the id of the next TransportVersion constant, do the following: * - Use the same major version, unless bumping majors * - Bump the server version part by 1, unless creating a patch version - * - Leave the serverless part as 00 + * - Leave the subsidiary part as 0 * - Bump the patch part if creating a patch version * * If a patch version is created, it should be placed sorted among the other existing constants. diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index c6a696e1f717a..0e31a02ea43ae 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -129,6 +129,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_26 = new Version(7_17_26_99); public static final Version V_7_17_27 = new Version(7_17_27_99); public static final Version V_7_17_28 = new Version(7_17_28_99); + public static final Version V_7_17_29 = new Version(7_17_29_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); @@ -196,9 +197,15 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_16_2 = new Version(8_16_02_99); public static final Version V_8_16_3 = new Version(8_16_03_99); public static final Version V_8_16_4 = new Version(8_16_04_99); + public static final Version V_8_16_5 = new Version(8_16_05_99); + public static final Version V_8_16_6 = new Version(8_16_06_99); + public static final Version V_8_16_7 = new Version(8_16_07_99); public static final Version V_8_17_0 = new Version(8_17_00_99); public static final Version V_8_17_1 = new Version(8_17_01_99); public static final Version V_8_17_2 = new Version(8_17_02_99); + public static final Version V_8_17_3 = new Version(8_17_03_99); + public static final Version V_8_17_4 = new Version(8_17_04_99); + public static final Version V_8_17_5 = new Version(8_17_05_99); public static final Version V_8_18_0 = new Version(8_18_00_99); public static final Version CURRENT = V_8_18_0; diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java index a158669d936fe..5b577eebcf6b0 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java @@ -390,7 +390,9 @@ static ActionListener assertOnce(ActionListener d private void assertFirstRun() { var previousRun = firstCompletion.compareAndExchange(null, new ElasticsearchException("executed already")); - assert previousRun == null : "[" + delegate + "] " + previousRun; // reports the stack traces of both completions + assert previousRun == null + // reports the stack traces of both completions + : new AssertionError("[" + delegate + "]", previousRun); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index eb0a005dc83ee..90548043ec6bd 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -26,10 +26,6 @@ import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction; import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction; import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; @@ -283,7 +279,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction; import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction; import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction; -import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction; @@ -297,7 +292,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction; import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction; import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction; -import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction; import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction; import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction; @@ -671,10 +665,7 @@ public void reg actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class); actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class); actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class); - actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class); - actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class); actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class); - actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class); actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class); @@ -872,8 +863,6 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< registerHandler.accept(new RestSnapshotsStatusAction()); registerHandler.accept(new RestSnapshottableFeaturesAction()); registerHandler.accept(new RestResetFeatureStateAction()); - registerHandler.accept(new RestGetFeatureUpgradeStatusAction()); - registerHandler.accept(new RestPostFeatureUpgradeAction()); registerHandler.accept(new RestGetIndicesAction()); registerHandler.accept(new RestIndicesStatsAction()); registerHandler.accept(new RestIndicesSegmentsAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index 0d5793b550f5a..9b1fcfd208301 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -13,9 +13,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.SingleResultDeduplicator; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterState; @@ -28,6 +31,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; @@ -47,7 +51,7 @@ public class TransportGetAllocationStatsAction extends TransportMasterNodeReadAc public static final ActionType TYPE = new ActionType<>("cluster:monitor/allocation/stats"); - private final AllocationStatsService allocationStatsService; + private final SingleResultDeduplicator> allocationStatsSupplier; private final DiskThresholdSettings diskThresholdSettings; private final FeatureService featureService; @@ -69,9 +73,15 @@ public TransportGetAllocationStatsAction( actionFilters, TransportGetAllocationStatsAction.Request::new, TransportGetAllocationStatsAction.Response::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + // DIRECT is ok here because we fork the allocation stats computation onto a MANAGEMENT thread if needed, or else we return + // very cheaply. + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + final var managementExecutor = threadPool.executor(ThreadPool.Names.MANAGEMENT); + this.allocationStatsSupplier = new SingleResultDeduplicator<>( + threadPool.getThreadContext(), + l -> managementExecutor.execute(ActionRunnable.supply(l, allocationStatsService::stats)) ); - this.allocationStatsService = allocationStatsService; this.diskThresholdSettings = new DiskThresholdSettings(clusterService.getSettings(), clusterService.getClusterSettings()); this.featureService = featureService; } @@ -88,15 +98,21 @@ protected void doExecute(Task task, Request request, ActionListener li @Override protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { - listener.onResponse( - new Response( - request.metrics().contains(Metric.ALLOCATIONS) ? allocationStatsService.stats() : Map.of(), + // NB we are still on a transport thread here - if adding more functionality here make sure to fork to a different pool + + final SubscribableListener> allocationStatsStep = request.metrics().contains(Metric.ALLOCATIONS) + ? SubscribableListener.newForked(allocationStatsSupplier::execute) + : SubscribableListener.newSucceeded(Map.of()); + + allocationStatsStep.andThenApply( + allocationStats -> new Response( + allocationStats, request.metrics().contains(Metric.FS) && featureService.clusterHasFeature(clusterService.state(), AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS) ? diskThresholdSettings : null ) - ); + ).addListener(listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java index 0d06da7f33483..3c059dfacb3ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java @@ -94,6 +94,7 @@ protected void masterOperation( return; } var clusterInfo = clusterInfoService.getClusterInfo(); + writeLoadForecaster.refreshLicense(); listener.onResponse( new DesiredBalanceResponse( desiredBalanceShardsAllocator.getStats(), diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index c84df0ddfe644..3b773ae686845 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -108,7 +108,7 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation( Task task ) { // We default to using an empty string as the keystore password so that we mimic pre 7.3 API behavior - try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) { + try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configDir())) { // reread keystore from config file if (keystore == null) { return new NodesReloadSecureSettingsResponse.NodeResponse( diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index b99059e6711ca..ac3da1db420a5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -350,9 +350,9 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public String getDescription() { final StringBuilder stringBuilder = new StringBuilder("repositories["); - Strings.collectionToDelimitedStringWithLimit(Arrays.asList(repositories), ",", "", "", 512, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(Arrays.asList(repositories), ",", 512, stringBuilder); stringBuilder.append("], snapshots["); - Strings.collectionToDelimitedStringWithLimit(Arrays.asList(snapshots), ",", "", "", 1024, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(Arrays.asList(snapshots), ",", 1024, stringBuilder); stringBuilder.append("]"); return stringBuilder.toString(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java index 61b9b00e1a3f2..eca2f585edd27 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java @@ -114,7 +114,7 @@ public int hashCode() { @Override public String getDescription() { final StringBuilder stringBuilder = new StringBuilder("shard").append(shardId).append(", repositories["); - Strings.collectionToDelimitedStringWithLimit(repositories, ",", "", "", 1024, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(repositories, ",", 1024, stringBuilder); stringBuilder.append("]"); return stringBuilder.toString(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index e74cbe9997da6..0b583e35c1eb3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -145,7 +145,7 @@ public boolean ignoreUnavailable() { @Override public String getDescription() { final StringBuilder stringBuilder = new StringBuilder("repository[").append(repository).append("], snapshots["); - Strings.collectionToDelimitedStringWithLimit(Arrays.asList(snapshots), ",", "", "", 1024, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(Arrays.asList(snapshots), ",", 1024, stringBuilder); stringBuilder.append("]"); return stringBuilder.toString(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java index 69ab9f57d2be7..071e9b42752c0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java @@ -77,6 +77,17 @@ public boolean hasErrors() { return errors; } + /** + * Get a list of all errors from the response. If there are no errors, an empty list is returned. + */ + public List getErrors() { + if (errors == false) { + return List.of(); + } else { + return actionResults.stream().filter(a -> a.getError() != null).map(AliasActionResult::getError).toList(); + } + } + /** * Build a response from a list of action results. Sets the errors boolean based * on whether an of the individual results contain an error. @@ -165,6 +176,13 @@ public static AliasActionResult buildSuccess(List indices, AliasActions return new AliasActionResult(indices, action, null); } + /** + * The error result if the action failed, null if the action succeeded. + */ + public ElasticsearchException getError() { + return error; + } + private int getStatus() { return error == null ? 200 : error.status().getStatus(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index cc96954c8a8e4..dba86df160955 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -410,16 +410,17 @@ public CreateIndexRequest source(Map source, DeprecationHandler depre for (Map.Entry entry : source.entrySet()) { String name = entry.getKey(); if (SETTINGS.match(name, deprecationHandler)) { - if (entry.getValue() instanceof Map == false) { - throw new ElasticsearchParseException("key [settings] must be an object"); - } + validateIsMap(SETTINGS.getPreferredName(), entry.getValue()); settings((Map) entry.getValue()); } else if (MAPPINGS.match(name, deprecationHandler)) { + validateIsMap(MAPPINGS.getPreferredName(), entry.getValue()); Map mappings = (Map) entry.getValue(); for (Map.Entry entry1 : mappings.entrySet()) { + validateIsMap(entry1.getKey(), entry1.getValue()); mapping(entry1.getKey(), (Map) entry1.getValue()); } } else if (ALIASES.match(name, deprecationHandler)) { + validateIsMap(ALIASES.getPreferredName(), entry.getValue()); aliases((Map) entry.getValue()); } else { throw new ElasticsearchParseException("unknown key [{}] for create index", name); @@ -428,6 +429,12 @@ public CreateIndexRequest source(Map source, DeprecationHandler depre return this; } + static void validateIsMap(String key, Object value) { + if (value instanceof Map == false) { + throw new ElasticsearchParseException("key [{}] must be an object", key); + } + } + public String mappings() { return this.mappings; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 423dffc6f4038..1ab5fbb568392 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -142,7 +142,7 @@ protected void masterOperation( listener.onFailure(new IllegalStateException(message)); return; } - updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor); + updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor); } else { updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt); } @@ -174,30 +174,54 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest( .waitForActiveShards(request.waitForActiveShards()); } - private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest( + private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest( CreateIndexRequest request, String cause, SystemIndexDescriptor descriptor ) { - final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY); + boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE) + && request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX); + final Settings settings; + final String mappings; final Set aliases; - if (descriptor.getAliasName() == null) { + final String indexName; + + // if we are migrating a system index to a new index, we use settings/mappings/index name from the request, + // since it was created by SystemIndexMigrator + if (indexMigrationInProgress) { + settings = request.settings(); + mappings = request.mappings(); + indexName = request.index(); + // we will update alias later on aliases = Set.of(); } else { - aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true)); - } + settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY); + mappings = descriptor.getMappings(); + + if (descriptor.getAliasName() == null) { + aliases = Set.of(); + } else { + aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true)); + } - // Throw an error if we are trying to directly create a system index other than the primary system index (or the alias) - if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) { - throw new IllegalArgumentException( - "Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex() - ); + // Throw an error if we are trying to directly create a system index other + // than the primary system index (or the alias, or we are migrating the index) + if (request.index().equals(descriptor.getPrimaryIndex()) == false + && request.index().equals(descriptor.getAliasName()) == false) { + throw new IllegalArgumentException( + "Cannot create system index with name " + + request.index() + + "; descriptor primary index is " + + descriptor.getPrimaryIndex() + ); + } + indexName = descriptor.getPrimaryIndex(); } - return new CreateIndexClusterStateUpdateRequest(cause, descriptor.getPrimaryIndex(), request.index()).aliases(aliases) + return new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()).aliases(aliases) .waitForActiveShards(ActiveShardCount.ALL) - .mappings(descriptor.getMappings()) + .mappings(mappings) .settings(settings); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 2870a6538f8bb..1c99d84900866 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -36,7 +36,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -126,7 +125,7 @@ protected void masterOperation( performMappingUpdate(concreteIndices, request, listener, metadataMappingService, false); } catch (IndexNotFoundException ex) { - logger.debug(() -> "failed to put mappings on indices [" + Arrays.asList(request.indices() + "]"), ex); + logger.debug(() -> "failed to put mappings on indices " + Arrays.toString(request.indices()), ex); throw ex; } } @@ -162,25 +161,21 @@ static void performMappingUpdate( MetadataMappingService metadataMappingService, boolean autoUpdate ) { - final ActionListener wrappedListener = listener.delegateResponse((l, e) -> { - logger.debug(() -> "failed to put mappings on indices [" + Arrays.asList(concreteIndices) + "]", e); + ActionListener.run(listener.delegateResponse((l, e) -> { + logger.debug(() -> "failed to put mappings on indices " + Arrays.toString(concreteIndices), e); l.onFailure(e); - }); - final PutMappingClusterStateUpdateRequest updateRequest; - try { - updateRequest = new PutMappingClusterStateUpdateRequest( - request.masterNodeTimeout(), - request.ackTimeout(), - request.source(), - autoUpdate, - concreteIndices - ); - } catch (IOException e) { - wrappedListener.onFailure(e); - return; - } - - metadataMappingService.putMapping(updateRequest, wrappedListener); + }), + wrappedListener -> metadataMappingService.putMapping( + new PutMappingClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), + request.source(), + autoUpdate, + concreteIndices + ), + wrappedListener + ) + ); } static String checkForFailureStoreViolations(ClusterState clusterState, Index[] concreteIndices, PutMappingRequest request) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 0f1b77af0242e..d61901b246d05 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -646,10 +646,6 @@ private static void enrichIndexAbstraction( : switch (resolvedExpression.selector()) { case DATA -> dataStream.getDataComponent().getIndices().stream(); case FAILURES -> dataStream.getFailureIndices().stream(); - case ALL_APPLICABLE -> Stream.concat( - dataStream.getIndices().stream(), - dataStream.getFailureIndices().stream() - ); }; String[] backingIndices = dataStreamIndices.map(Index::getName).toArray(String[]::new); dataStreams.add(new ResolvedDataStream(dataStream.getName(), backingIndices, DataStream.TIMESTAMP_FIELD_NAME)); @@ -670,13 +666,6 @@ private static Stream getAliasIndexStream(ResolvedExpression resolvedExpr assert ia.isDataStreamRelated() : "Illegal selector [failures] used on non data stream alias"; yield ia.getFailureIndices(metadata).stream(); } - case ALL_APPLICABLE -> { - if (ia.isDataStreamRelated()) { - yield Stream.concat(ia.getIndices().stream(), ia.getFailureIndices(metadata).stream()); - } else { - yield ia.getIndices().stream(); - } - } }; } return aliasIndices; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java index be08293fe90db..f30bdfe9df09c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java @@ -32,7 +32,6 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.CancellableTask; @@ -186,7 +185,7 @@ record LazyRolloverExecutor( @Override public ClusterState execute(BatchExecutionContext batchExecutionContext) { final var listener = new AllocationActionMultiListener(threadPool.getThreadContext()); - final var results = new ArrayList(batchExecutionContext.taskContexts().size()); + final var results = new ArrayList(batchExecutionContext.taskContexts().size()); var state = batchExecutionContext.initialState(); Map>> groupedRequests = new HashMap<>(); for (final var taskContext : batchExecutionContext.taskContexts()) { @@ -205,15 +204,9 @@ public ClusterState execute(BatchExecutionContext batchExecuti } if (state != batchExecutionContext.initialState()) { - var reason = new StringBuilder(); - Strings.collectionToDelimitedStringWithLimit( - (Iterable) () -> Iterators.map(results.iterator(), t -> t.sourceIndexName() + "->" + t.rolloverIndexName()), - ",", - "lazy bulk rollover [", - "]", - 1024, - reason - ); + var reason = new StringBuilder("lazy bulk rollover ["); + Strings.collectionToDelimitedStringWithLimit(results, ",", 1024, reason); + reason.append(']'); try (var ignored = batchExecutionContext.dropHeadersContext()) { state = allocationService.reroute(state, reason.toString(), listener.reroute()); } @@ -226,7 +219,7 @@ public ClusterState execute(BatchExecutionContext batchExecuti public ClusterState executeTask( ClusterState currentState, RolloverRequest rolloverRequest, - List results, + ArrayList results, List> rolloverTaskContexts, AllocationActionMultiListener allocationActionMultiListener ) throws Exception { @@ -263,11 +256,16 @@ public ClusterState executeTask( null, isFailureStoreRollover ); - results.add(rolloverResult); + results.add(rolloverResult.sourceIndexName() + "->" + rolloverResult.rolloverIndexName()); logger.trace("lazy rollover result [{}]", rolloverResult); final var rolloverIndexName = rolloverResult.rolloverIndexName(); final var sourceIndexName = rolloverResult.sourceIndexName(); + logger.info( + "rolling over data stream [{}] to index [{}] because it was marked for lazy rollover", + dataStream.getName(), + rolloverIndexName + ); final var waitForActiveShardsTimeout = rolloverRequest.masterNodeTimeout().millis() < 0 ? null diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 0c22a17bb1f6b..eecb0ff354ba5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.datastreams.autosharding.AutoShardingResult; @@ -148,7 +149,7 @@ public RolloverResult rolloverClusterState( @Nullable AutoShardingResult autoShardingResult, boolean isFailureStoreRollover ) throws Exception { - validate(currentState.metadata(), rolloverTarget, newIndexName, createIndexRequest, isFailureStoreRollover); + validate(currentState.metadata(), rolloverTarget, newIndexName, createIndexRequest); final IndexAbstraction indexAbstraction = currentState.metadata().getIndicesLookup().get(rolloverTarget); return switch (indexAbstraction.getType()) { case ALIAS -> rolloverAlias( @@ -190,7 +191,7 @@ public static NameResolution resolveRolloverNames( CreateIndexRequest createIndexRequest, boolean isFailureStoreRollover ) { - validate(currentState.metadata(), rolloverTarget, newIndexName, createIndexRequest, isFailureStoreRollover); + validate(currentState.metadata(), rolloverTarget, newIndexName, createIndexRequest); final IndexAbstraction indexAbstraction = currentState.metadata().getIndicesLookup().get(rolloverTarget); return switch (indexAbstraction.getType()) { case ALIAS -> resolveAliasRolloverNames(currentState.metadata(), indexAbstraction, newIndexName); @@ -430,6 +431,7 @@ yield new DataStreamAutoShardingEvent( ); } + writeLoadForecaster.refreshLicense(); metadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, metadataBuilder); metadataBuilder = withShardSizeForecastForWriteIndex(dataStreamName, metadataBuilder); @@ -626,16 +628,10 @@ static void checkNoDuplicatedAliasInIndexTemplate( } } - static void validate( - Metadata metadata, - String rolloverTarget, - String newIndexName, - CreateIndexRequest request, - boolean isFailureStoreRollover - ) { + static void validate(Metadata metadata, String rolloverTarget, String newIndexName, CreateIndexRequest request) { final IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(rolloverTarget); if (indexAbstraction == null) { - throw new IllegalArgumentException("rollover target [" + rolloverTarget + "] does not exist"); + throw new ResourceNotFoundException("rollover target [" + rolloverTarget + "] does not exist"); } if (VALID_ROLLOVER_TARGETS.contains(indexAbstraction.getType()) == false) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 7b0294f881099..7bea30df4c02f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -13,15 +13,14 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -149,14 +148,12 @@ public ActionRequestValidationException validate() { ); } + // Ensure we have a valid selector in the request if (rolloverTarget != null) { - ResolvedExpression resolvedExpression = SelectorResolver.parseExpression(rolloverTarget, indicesOptions); - IndexComponentSelector selector = resolvedExpression.selector(); - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - validationException = addValidationError( - "rollover cannot be applied to both regular and failure indices at the same time", - validationException - ); + try { + SelectorResolver.parseExpression(rolloverTarget, indicesOptions); + } catch (InvalidIndexNameException exception) { + validationException = addValidationError(exception.getMessage(), validationException); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 8a6e84645a92f..59feb9b7e0bbb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -47,7 +47,6 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; @@ -154,12 +153,33 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState .build(), IndicesOptions.GatekeeperOptions.DEFAULT ); - - return state.blocks() - .indicesBlockedException( - ClusterBlockLevel.METADATA_WRITE, - indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request) - ); + ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(request.getRolloverTarget(), request.indicesOptions()); + final IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(resolvedRolloverTarget.resource()); + final String[] indicesToCheck; + if (indexAbstraction != null && indexAbstraction.getType().equals(IndexAbstraction.Type.DATA_STREAM)) { + DataStream dataStream = (DataStream) indexAbstraction; + boolean targetFailureStore = resolvedRolloverTarget.selector() != null + && resolvedRolloverTarget.selector().shouldIncludeFailures(); + if (targetFailureStore == false) { + assert dataStream.getWriteIndex() != null : dataStream.getName() + " is a data stream but has no write index"; + assert dataStream.getWriteIndex().getName() != null + : dataStream.getName() + " is a data stream but the write index is null"; + indicesToCheck = new String[] { dataStream.getWriteIndex().getName() }; + } else if (dataStream.getWriteFailureIndex() != null) { + assert dataStream.getWriteFailureIndex().getName() != null + : "the write index for the data stream " + dataStream.getName() + " is null"; + indicesToCheck = new String[] { dataStream.getWriteFailureIndex().getName() }; + } else { + indicesToCheck = null; + } + } else { + indicesToCheck = indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request); + } + if (indicesToCheck == null) { + return null; + } else { + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indicesToCheck); + } } @Override @@ -192,44 +212,17 @@ protected void masterOperation( final String trialRolloverIndexName = trialRolloverNames.rolloverName(); MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, metadata, clusterState.routingTable()); - boolean isDataStream = metadata.dataStreams().containsKey(resolvedRolloverTarget.resource()); if (rolloverRequest.isLazy()) { - if (isDataStream == false || rolloverRequest.getConditions().hasConditions()) { - String message; - if (isDataStream) { - message = "Lazy rollover can be used only without any conditions." - + " Please remove the conditions from the request body or the query parameter 'lazy'."; - } else if (rolloverRequest.getConditions().hasConditions() == false) { - message = "Lazy rollover can be applied only on a data stream." + " Please remove the query parameter 'lazy'."; - } else { - message = "Lazy rollover can be applied only on a data stream with no conditions." - + " Please remove the query parameter 'lazy'."; - } - listener.onFailure(new IllegalArgumentException(message)); - return; - } - if (rolloverRequest.isDryRun() == false) { - metadataDataStreamsService.setRolloverOnWrite( - resolvedRolloverTarget.resource(), - true, - targetFailureStore, - rolloverRequest.ackTimeout(), - rolloverRequest.masterNodeTimeout(), - listener.map( - response -> new RolloverResponse( - trialSourceIndexName, - trialRolloverIndexName, - Map.of(), - false, - false, - response.isAcknowledged(), - false, - response.isAcknowledged() - ) - ) - ); - return; - } + markForLazyRollover( + rolloverRequest, + listener, + metadata, + resolvedRolloverTarget, + targetFailureStore, + trialSourceIndexName, + trialRolloverIndexName + ); + return; } final IndexAbstraction rolloverTargetAbstraction = clusterState.metadata() @@ -325,7 +318,7 @@ protected void masterOperation( false, false, false, - rolloverRequest.isLazy() + false ); // If this is a dry run, return with the results without invoking a cluster state update @@ -353,6 +346,57 @@ protected void masterOperation( ); } + private void markForLazyRollover( + RolloverRequest rolloverRequest, + ActionListener listener, + Metadata metadata, + ResolvedExpression resolvedRolloverTarget, + boolean targetFailureStore, + String trialSourceIndexName, + String trialRolloverIndexName + ) { + boolean isDataStream = metadata.dataStreams().containsKey(resolvedRolloverTarget.resource()); + if (isDataStream == false || rolloverRequest.getConditions().hasConditions()) { + String message; + if (isDataStream) { + message = "Lazy rollover can be used only without any conditions." + + " Please remove the conditions from the request body or the query parameter 'lazy'."; + } else if (rolloverRequest.getConditions().hasConditions() == false) { + message = "Lazy rollover can be applied only on a data stream. Please remove the query parameter 'lazy'."; + } else { + message = "Lazy rollover can be applied only on a data stream with no conditions." + + " Please remove the query parameter 'lazy'."; + } + listener.onFailure(new IllegalArgumentException(message)); + return; + } + if (rolloverRequest.isDryRun()) { + listener.onResponse( + new RolloverResponse(trialSourceIndexName, trialRolloverIndexName, Map.of(), true, false, false, false, true) + ); + return; + } + metadataDataStreamsService.setRolloverOnWrite( + resolvedRolloverTarget.resource(), + true, + targetFailureStore, + rolloverRequest.ackTimeout(), + rolloverRequest.masterNodeTimeout(), + listener.map( + response -> new RolloverResponse( + trialSourceIndexName, + trialRolloverIndexName, + Map.of(), + false, + false, + response.isAcknowledged(), + false, + true + ) + ) + ); + } + private void initializeFailureStore( RolloverRequest rolloverRequest, ActionListener listener, @@ -462,7 +506,7 @@ record RolloverExecutor( @Override public ClusterState execute(BatchExecutionContext batchExecutionContext) { final var listener = new AllocationActionMultiListener(threadPool.getThreadContext()); - final var results = new ArrayList(batchExecutionContext.taskContexts().size()); + final var results = new ArrayList(batchExecutionContext.taskContexts().size()); var state = batchExecutionContext.initialState(); for (final var taskContext : batchExecutionContext.taskContexts()) { try (var ignored = taskContext.captureResponseHeaders()) { @@ -473,15 +517,9 @@ public ClusterState execute(BatchExecutionContext batchExecutionCo } if (state != batchExecutionContext.initialState()) { - var reason = new StringBuilder(); - Strings.collectionToDelimitedStringWithLimit( - (Iterable) () -> Iterators.map(results.iterator(), t -> t.sourceIndexName() + "->" + t.rolloverIndexName()), - ",", - "bulk rollover [", - "]", - 1024, - reason - ); + var reason = new StringBuilder("bulk rollover ["); + Strings.collectionToDelimitedStringWithLimit(results, ",", 1024, reason); + reason.append(']'); try (var ignored = batchExecutionContext.dropHeadersContext()) { state = allocationService.reroute(state, reason.toString(), listener.reroute()); } @@ -493,7 +531,7 @@ public ClusterState execute(BatchExecutionContext batchExecutionCo public ClusterState executeTask( ClusterState currentState, - List results, + ArrayList results, TaskContext rolloverTaskContext, AllocationActionMultiListener allocationActionMultiListener ) throws Exception { @@ -565,7 +603,7 @@ public ClusterState executeTask( rolloverTask.autoShardingResult(), targetFailureStore ); - results.add(rolloverResult); + results.add(rolloverResult.sourceIndexName() + "->" + rolloverResult.rolloverIndexName()); logger.trace("rollover result [{}]", rolloverResult); final var rolloverIndexName = rolloverResult.rolloverIndexName(); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java index 7a3317eafac4f..bb17638feae5b 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java @@ -10,6 +10,7 @@ package org.elasticsearch.action.datastreams; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndicesOptions.WildcardOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; @@ -40,9 +41,10 @@ public static List getDataStreamNames( } public static IndicesOptions updateIndicesOptions(IndicesOptions indicesOptions) { + // if expandWildcardsOpen=false, then it will be overridden to true if (indicesOptions.expandWildcardsOpen() == false) { indicesOptions = IndicesOptions.builder(indicesOptions) - .wildcardOptions(IndicesOptions.WildcardOptions.builder(indicesOptions.wildcardOptions()).matchOpen(true)) + .wildcardOptions(WildcardOptions.builder(indicesOptions.wildcardOptions()).matchOpen(true)) .build(); } return indicesOptions; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java index df9e6aa8f58da..565f2491e5d01 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java @@ -158,7 +158,7 @@ public ActionRequestValidationException validate() { @Override public String getDescription() { final StringBuilder stringBuilder = new StringBuilder("shards["); - Strings.collectionToDelimitedStringWithLimit(shardIds, ",", "", "", 1024, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(shardIds, ",", 1024, stringBuilder); return completeDescription(stringBuilder, fields, filters, allowedTypes, includeEmptyFields); } @@ -170,11 +170,11 @@ static String completeDescription( boolean includeEmptyFields ) { stringBuilder.append("], fields["); - Strings.collectionToDelimitedStringWithLimit(Arrays.asList(fields), ",", "", "", 1024, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(Arrays.asList(fields), ",", 1024, stringBuilder); stringBuilder.append("], filters["); - Strings.collectionToDelimitedString(Arrays.asList(filters), ",", "", "", stringBuilder); + Strings.collectionToDelimitedString(Arrays.asList(filters), ",", stringBuilder); stringBuilder.append("], types["); - Strings.collectionToDelimitedString(Arrays.asList(allowedTypes), ",", "", "", stringBuilder); + Strings.collectionToDelimitedString(Arrays.asList(allowedTypes), ",", stringBuilder); stringBuilder.append("], includeEmptyFields["); stringBuilder.append(includeEmptyFields); stringBuilder.append("]"); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 80cdbf9d3d867..88eb2ef4fb13d 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -289,7 +289,7 @@ public int hashCode() { @Override public String getDescription() { final StringBuilder stringBuilder = new StringBuilder("indices["); - Strings.collectionToDelimitedStringWithLimit(Arrays.asList(indices), ",", "", "", 1024, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(Arrays.asList(indices), ",", 1024, stringBuilder); return FieldCapabilitiesNodeRequest.completeDescription(stringBuilder, fields, filters, types, includeEmptyFields); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 83ee6c216ad49..d67d5d2faf47e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.DelayableWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -91,7 +92,15 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO public SearchResponse(StreamInput in) throws IOException { super(in); this.hits = SearchHits.readFrom(in, true); - this.aggregations = in.readBoolean() ? InternalAggregations.readFrom(in) : null; + if (in.readBoolean()) { + // deserialize the aggregations trying to deduplicate the object created + // TODO: use DelayableWriteable instead. + this.aggregations = InternalAggregations.readFrom( + DelayableWriteable.wrapWithDeduplicatorStreamInput(in, in.getTransportVersion(), in.namedWriteableRegistry()) + ); + } else { + this.aggregations = null; + } this.suggest = in.readBoolean() ? new Suggest(in) : null; this.timedOut = in.readBoolean(); this.terminatedEarly = in.readOptionalBoolean(); diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java index 910be151d1bf5..4ead590a99d90 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.support; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -23,14 +24,11 @@ * We define as index components the two different sets of indices a data stream could consist of: * - DATA: represents the backing indices * - FAILURES: represent the failing indices - * - ALL: represents all available in this expression components, meaning if it's a data stream both backing and failure indices and if it's - * an index only the index itself. * Note: An index is its own DATA component, but it cannot have a FAILURE component. */ public enum IndexComponentSelector implements Writeable { DATA("data", (byte) 0), - FAILURES("failures", (byte) 1), - ALL_APPLICABLE("*", (byte) 2); + FAILURES("failures", (byte) 1); private final String key; private final byte id; @@ -75,7 +73,13 @@ public static IndexComponentSelector getByKey(String key) { } public static IndexComponentSelector read(StreamInput in) throws IOException { - return getById(in.readByte()); + byte id = in.readByte(); + if (in.getTransportVersion().onOrAfter(TransportVersions.REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_18)) { + return getById(id); + } else { + // Legacy value ::*, converted to ::data + return id == 2 ? DATA : getById(id); + } } // Visible for testing @@ -95,10 +99,10 @@ public void writeTo(StreamOutput out) throws IOException { } public boolean shouldIncludeData() { - return this == ALL_APPLICABLE || this == DATA; + return this == DATA; } public boolean shouldIncludeFailures() { - return this == ALL_APPLICABLE || this == FAILURES; + return this == FAILURES; } } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java index 8f1537d917c15..f4363ce1948b9 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java @@ -24,7 +24,7 @@ private BootstrapUtil() {} public static SecureSettings loadSecureSettings(Environment initialEnv, SecureString keystorePassword) throws BootstrapException { try { - return KeyStoreWrapper.bootstrap(initialEnv.configFile(), () -> keystorePassword); + return KeyStoreWrapper.bootstrap(initialEnv.configDir(), () -> keystorePassword); } catch (Exception e) { throw new BootstrapException(e); } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java index 7b85b369b5dd5..54244f320840a 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java @@ -52,7 +52,7 @@ static Supplier buildConsoleLoader(ClassLoader classLoader) { } private static ClassLoader buildClassLoader(Environment env) { - final Path libDir = env.libFile().resolve("tools").resolve("ansi-console"); + final Path libDir = env.libDir().resolve("tools").resolve("ansi-console"); try (var libDirFilesStream = Files.list(libDir)) { final URL[] urls = libDirFilesStream.filter(each -> each.getFileName().toString().endsWith(".jar")) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 56b49fcee5895..6d828301a3420 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -28,13 +28,15 @@ import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.core.AbstractRefCounted; -import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; -import org.elasticsearch.entitlement.runtime.policy.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.entitlement.runtime.policy.Policy; -import org.elasticsearch.entitlement.runtime.policy.PolicyParserUtils; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; +import org.elasticsearch.entitlement.runtime.policy.PolicyUtils; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.jdk.JarHell; @@ -46,12 +48,14 @@ import org.elasticsearch.nativeaccess.NativeAccess; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.PluginBundle; import org.elasticsearch.plugins.PluginsLoader; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.lang.invoke.MethodHandles; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.security.Permission; @@ -65,6 +69,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.bootstrap.BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING; @@ -75,6 +80,9 @@ */ class Elasticsearch { + private static final String POLICY_PATCH_PREFIX = "es.entitlements.policy."; + private static final String SERVER_POLICY_PATCH_NAME = POLICY_PATCH_PREFIX + "server"; + /** * Main entry point for starting elasticsearch. */ @@ -115,9 +123,8 @@ private static Bootstrap initPhase1() { final PrintStream out = getStdout(); final PrintStream err = getStderr(); final ServerArgs args; - final boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(System.getProperty("es.entitlements.enabled", "false")); - // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly - final boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled; + + final boolean useEntitlements = true; try { initSecurityProperties(); @@ -185,7 +192,7 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { nodeEnv.validateNativesConfig(); // temporary directories are important for JNA initializeNatives( - nodeEnv.tmpFile(), + nodeEnv.tmpDir(), BootstrapSettings.MEMORY_LOCK_SETTING.get(args.nodeSettings()), true, // always install system call filters, not user-configurable since 8.0.0 BootstrapSettings.CTRLHANDLER_SETTING.get(args.nodeSettings()) @@ -217,8 +224,8 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { ); // load the plugin Java modules and layers now for use in entitlements - var modulesBundles = PluginsLoader.loadModulesBundles(nodeEnv.modulesFile()); - var pluginsBundles = PluginsLoader.loadPluginsBundles(nodeEnv.pluginsFile()); + var modulesBundles = PluginsLoader.loadModulesBundles(nodeEnv.modulesDir()); + var pluginsBundles = PluginsLoader.loadPluginsBundles(nodeEnv.pluginsDir()); final PluginsLoader pluginsLoader; @@ -227,17 +234,46 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { var pluginData = Stream.concat( modulesBundles.stream() - .map(bundle -> new PolicyParserUtils.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)), + .map(bundle -> new PolicyUtils.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)), pluginsBundles.stream() - .map(bundle -> new PolicyParserUtils.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true)) + .map(bundle -> new PolicyUtils.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true)) ).toList(); - var pluginPolicies = PolicyParserUtils.createPluginPolicies(pluginData); + + var pluginPolicyPatches = collectPluginPolicyPatches(modulesBundles, pluginsBundles, logger); + var pluginPolicies = PolicyUtils.createPluginPolicies(pluginData, pluginPolicyPatches, Build.current().version()); + var serverPolicyPatch = PolicyUtils.parseEncodedPolicyIfExists( + System.getProperty(SERVER_POLICY_PATCH_NAME), + Build.current().version(), + false, + "server", + PolicyManager.SERVER_LAYER_MODULES.stream().map(Module::getName).collect(Collectors.toUnmodifiableSet()) + ); pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, findPluginsWithNativeAccess(pluginPolicies)); var pluginsResolver = PluginsResolver.create(pluginsLoader); - EntitlementBootstrap.bootstrap(pluginPolicies, pluginsResolver::resolveClassToPluginName); - } else if (RuntimeVersionFeature.isSecurityManagerAvailable()) { + Map sourcePaths = Stream.concat(modulesBundles.stream(), pluginsBundles.stream()) + .collect(Collectors.toUnmodifiableMap(bundle -> bundle.pluginDescriptor().getName(), PluginBundle::getDir)); + EntitlementBootstrap.bootstrap( + serverPolicyPatch, + pluginPolicies, + pluginsResolver::resolveClassToPluginName, + nodeEnv.settings()::getValues, + nodeEnv.dataDirs(), + nodeEnv.repoDirs(), + nodeEnv.configDir(), + nodeEnv.libDir(), + nodeEnv.modulesDir(), + nodeEnv.pluginsDir(), + sourcePaths, + nodeEnv.logsDir(), + nodeEnv.tmpDir(), + args.pidFile(), + Set.of(EntitlementSelfTester.class) + ); + EntitlementSelfTester.entitlementSelfTest(); + } else { + assert RuntimeVersionFeature.isSecurityManagerAvailable(); // no need to explicitly enable native access for legacy code pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, Map.of()); // install SM after natives, shutdown hooks, etc. @@ -247,15 +283,73 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(args.nodeSettings()), args.pidFile() ); - } else { - // TODO: should we throw/interrupt startup in this case? - pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, Map.of()); - LogManager.getLogger(Elasticsearch.class).warn("Bootstrapping without any protection"); } bootstrap.setPluginsLoader(pluginsLoader); } + private static Map collectPluginPolicyPatches( + Set modulesBundles, + Set pluginsBundles, + Logger logger + ) { + var policyPatches = new HashMap(); + var systemProperties = BootstrapInfo.getSystemProperties(); + systemProperties.keys().asIterator().forEachRemaining(key -> { + var value = systemProperties.get(key); + if (key instanceof String k + && value instanceof String v + && k.startsWith(POLICY_PATCH_PREFIX) + && k.equals(SERVER_POLICY_PATCH_NAME) == false) { + policyPatches.put(k.substring(POLICY_PATCH_PREFIX.length()), v); + } + }); + var pluginNames = Stream.concat(modulesBundles.stream(), pluginsBundles.stream()) + .map(bundle -> bundle.pluginDescriptor().getName()) + .collect(Collectors.toUnmodifiableSet()); + + for (var patchedPluginName : policyPatches.keySet()) { + if (pluginNames.contains(patchedPluginName) == false) { + logger.warn( + "Found command-line policy patch for unknown plugin [{}] (available plugins: [{}])", + patchedPluginName, + String.join(", ", pluginNames) + ); + } + } + return policyPatches; + } + + private static class EntitlementSelfTester { + // check entitlements were loaded correctly. note this must be outside the entitlements lib. + private static void entitlementSelfTest() { + ensureCannotStartProcess(ProcessBuilder::start); + // Try again with reflection + ensureCannotStartProcess(EntitlementSelfTester::reflectiveStartProcess); + } + + private static void ensureCannotStartProcess(CheckedConsumer startProcess) { + try { + // The command doesn't matter; it doesn't even need to exist + startProcess.accept(new ProcessBuilder("")); + } catch (NotEntitledException e) { + return; + } catch (Exception e) { + throw new IllegalStateException("Failed entitlement protection self-test", e); + } + throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); + } + + private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { + try { + var start = ProcessBuilder.class.getMethod("start"); + start.invoke(pb); + } catch (InvocationTargetException e) { + throw (Exception) e.getCause(); + } + } + } + private static void ensureInitialized(Class... classes) { for (final var clazz : classes) { try { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Security.java b/server/src/main/java/org/elasticsearch/bootstrap/Security.java index dc6de9a6b2c91..a352112b67afb 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -178,11 +178,11 @@ static Map getPluginAndModulePermissions(Environment environment) t } }; - for (Path plugin : PluginsUtils.findPluginDirs(environment.pluginsFile())) { - addPolicy.accept(PolicyUtil.getPluginPolicyInfo(plugin, environment.tmpFile())); + for (Path plugin : PluginsUtils.findPluginDirs(environment.pluginsDir())) { + addPolicy.accept(PolicyUtil.getPluginPolicyInfo(plugin, environment.tmpDir())); } - for (Path plugin : PluginsUtils.findPluginDirs(environment.modulesFile())) { - addPolicy.accept(PolicyUtil.getModulePolicyInfo(plugin, environment.tmpFile())); + for (Path plugin : PluginsUtils.findPluginDirs(environment.modulesDir())) { + addPolicy.accept(PolicyUtil.getModulePolicyInfo(plugin, environment.tmpDir())); } return Collections.unmodifiableMap(map); @@ -199,7 +199,7 @@ static Permissions createPermissions(Environment environment, Path pidFile) thro private static List createRecursiveDataPathPermission(Environment environment) throws IOException { Permissions policy = new Permissions(); - for (Path path : environment.dataFiles()) { + for (Path path : environment.dataDirs()) { addDirectoryPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete", true); } return toFilePermissions(policy); @@ -215,13 +215,13 @@ private static Map> readSecuredConfigFiles( Map> securedSettingKeys = new HashMap<>(); for (URL url : mainCodebases) { - for (Permission p : PolicyUtil.getPolicyPermissions(url, template, environment.tmpFile())) { + for (Permission p : PolicyUtil.getPolicyPermissions(url, template, environment.tmpDir())) { readSecuredConfigFilePermissions(environment, url, p, securedConfigFiles, securedSettingKeys); } } for (var pp : pluginPolicies.entrySet()) { - for (Permission p : PolicyUtil.getPolicyPermissions(pp.getKey(), pp.getValue(), environment.tmpFile())) { + for (Permission p : PolicyUtil.getPolicyPermissions(pp.getKey(), pp.getValue(), environment.tmpDir())) { readSecuredConfigFilePermissions(environment, pp.getKey(), p, securedConfigFiles, securedSettingKeys); } } @@ -242,8 +242,8 @@ private static Map> readSecuredConfigFiles( // If the setting shouldn't be an HTTPS URL, that'll be caught by that setting's validation later in the process. // HTTP (no S) URLs are not supported. if (settingValue.toLowerCase(Locale.ROOT).startsWith("https://") == false) { - Path file = environment.configFile().resolve(settingValue); - if (file.startsWith(environment.configFile()) == false) { + Path file = environment.configDir().resolve(settingValue); + if (file.startsWith(environment.configDir()) == false) { throw new IllegalStateException( ps.getValue() + " tried to grant access to file outside config directory " + file ); @@ -263,9 +263,9 @@ private static Map> readSecuredConfigFiles( // always add some config files as exclusive files that no one can access // there's no reason for anyone to read these once the security manager is initialized // so if something has tried to grant itself access, crash out with an error - addSpeciallySecuredConfigFile(securedConfigFiles, environment.configFile().resolve("elasticsearch.yml").toString()); - addSpeciallySecuredConfigFile(securedConfigFiles, environment.configFile().resolve("jvm.options").toString()); - addSpeciallySecuredConfigFile(securedConfigFiles, environment.configFile().resolve("jvm.options.d/-").toString()); + addSpeciallySecuredConfigFile(securedConfigFiles, environment.configDir().resolve("elasticsearch.yml").toString()); + addSpeciallySecuredConfigFile(securedConfigFiles, environment.configDir().resolve("jvm.options").toString()); + addSpeciallySecuredConfigFile(securedConfigFiles, environment.configDir().resolve("jvm.options.d/-").toString()); return Collections.unmodifiableMap(securedConfigFiles); } @@ -279,8 +279,8 @@ private static void readSecuredConfigFilePermissions( ) { String securedFileName = extractSecuredName(p, SecuredConfigFileAccessPermission.class); if (securedFileName != null) { - Path securedFile = environment.configFile().resolve(securedFileName); - if (securedFile.startsWith(environment.configFile()) == false) { + Path securedFile = environment.configDir().resolve(securedFileName); + if (securedFile.startsWith(environment.configDir()) == false) { throw new IllegalStateException("[" + url + "] tried to grant access to file outside config directory " + securedFile); } logger.debug("Jar {} securing access to config file {}", url, securedFile); @@ -336,26 +336,26 @@ static void addClasspathPermissions(Permissions policy) throws IOException { */ static void addFilePermissions(Permissions policy, Environment environment, Path pidFile) throws IOException { // read-only dirs - addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.binFile(), "read,readlink", false); - addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libFile(), "read,readlink", false); - addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesFile(), "read,readlink", false); - addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsFile(), "read,readlink", false); - addDirectoryPath(policy, "path.conf", environment.configFile(), "read,readlink", false); + addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.binDir(), "read,readlink", false); + addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libDir(), "read,readlink", false); + addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesDir(), "read,readlink", false); + addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsDir(), "read,readlink", false); + addDirectoryPath(policy, "path.conf", environment.configDir(), "read,readlink", false); // read-write dirs - addDirectoryPath(policy, "java.io.tmpdir", environment.tmpFile(), "read,readlink,write,delete", false); - addDirectoryPath(policy, Environment.PATH_LOGS_SETTING.getKey(), environment.logsFile(), "read,readlink,write,delete", false); - if (environment.sharedDataFile() != null) { + addDirectoryPath(policy, "java.io.tmpdir", environment.tmpDir(), "read,readlink,write,delete", false); + addDirectoryPath(policy, Environment.PATH_LOGS_SETTING.getKey(), environment.logsDir(), "read,readlink,write,delete", false); + if (environment.sharedDataDir() != null) { addDirectoryPath( policy, Environment.PATH_SHARED_DATA_SETTING.getKey(), - environment.sharedDataFile(), + environment.sharedDataDir(), "read,readlink,write,delete", false ); } final Set dataFilesPaths = new HashSet<>(); - for (Path path : environment.dataFiles()) { + for (Path path : environment.dataDirs()) { addDirectoryPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete", false); /* * We have to do this after adding the path because a side effect of that is that the directory is created; the Path#toRealPath @@ -371,7 +371,7 @@ static void addFilePermissions(Permissions policy, Environment environment, Path throw new IllegalStateException("unable to access [" + path + "]", e); } } - for (Path path : environment.repoFiles()) { + for (Path path : environment.repoDirs()) { addDirectoryPath(policy, Environment.PATH_REPO_SETTING.getKey(), path, "read,readlink,write,delete", false); } @@ -380,7 +380,7 @@ static void addFilePermissions(Permissions policy, Environment environment, Path addSingleFilePath(policy, pidFile, "delete"); } // we need to touch the operator/settings.json file when restoring from snapshots, on some OSs it needs file write permission - addSingleFilePath(policy, environment.configFile().resolve(OPERATOR_DIRECTORY).resolve(SETTINGS_FILE_NAME), "read,readlink,write"); + addSingleFilePath(policy, environment.configDir().resolve(OPERATOR_DIRECTORY).resolve(SETTINGS_FILE_NAME), "read,readlink,write"); } /** diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java index 6a4296d9b0478..35284cebf22ad 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java @@ -69,14 +69,14 @@ void spawnNativeControllers(final Environment environment) throws IOException { if (spawned.compareAndSet(false, true) == false) { throw new IllegalStateException("native controllers already spawned"); } - if (Files.exists(environment.modulesFile()) == false) { - throw new IllegalStateException("modules directory [" + environment.modulesFile() + "] not found"); + if (Files.exists(environment.modulesDir()) == false) { + throw new IllegalStateException("modules directory [" + environment.modulesDir() + "] not found"); } /* * For each module, attempt to spawn the controller daemon. Silently ignore any module that doesn't include a controller for the * correct platform. */ - List paths = PluginsUtils.findPluginDirs(environment.modulesFile()); + List paths = PluginsUtils.findPluginDirs(environment.modulesDir()); for (final Path modules : paths) { final PluginDescriptor info = PluginDescriptor.readFromProperties(modules); final Path spawnPath = Platforms.nativeControllerPath(modules); @@ -91,7 +91,7 @@ void spawnNativeControllers(final Environment environment) throws IOException { ); throw new IllegalArgumentException(message); } - final Process process = spawnNativeController(spawnPath, environment.tmpFile()); + final Process process = spawnNativeController(spawnPath, environment.tmpDir()); // The process _shouldn't_ write any output via its stdout or stderr, but if it does then // it will block if nothing is reading that output. To avoid this we can pipe the // outputs and create pump threads to write any messages there to the ES log. diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 5b87267b69527..972262119b485 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -84,7 +84,6 @@ import org.elasticsearch.tasks.TaskResultsService; import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.upgrades.FeatureMigrationResults; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -235,7 +234,6 @@ public static List getNamedWriteables() { ); registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom); registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom); - registerMetadataCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom); registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom); registerMetadataCustom( entries, diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index f7cad013554c6..a02eaf852d99c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -322,6 +322,28 @@ public boolean hasMixedSystemIndexVersions() { .anyMatch(e -> e.systemIndexMappingsVersion().equals(minVersions.systemIndexMappingsVersion()) == false); } + /** + * @return the minimum {@link TransportVersion} that will be used for all future intra-cluster node-to-node communications. This value + * only ever increases, so if {@code v.onOrAfter(cs.getMinTransportVersion())} is true once then it will remain true in the + * future. + *

+ * There are some subtle exceptions: + *

    + *
  • The "only ever increases" property is handled by the master node using the in-memory (ephemeral) part of the + * {@link ClusterState} only, so in theory a full restart of a mixed-version cluster may lose that state and allow some nodes to see + * this value decrease. For this to happen in practice requires some fairly unlucky timing during the initial master election. We + * tell users not to do this: if something breaks during a rolling upgrade then they should upgrade all remaining nodes to continue. + * But we do not enforce it. + *
  • The "used for all node-to-node communications" is false in a disordered upgrade (an upgrade to a semantically-newer but + * chronologically-older version) because for each connection between such nodes we will use {@link + * TransportVersion#bestKnownVersion} to pick a transport version which is known by both endpoints. We tell users not to do + * disordered upgrades too, but do not enforce it. + *
+ *

+ * Note also that node-to-node communications which are not intra-cluster (i.e. they are not between nodes in the same cluster) + * may sometimes use an earlier {@link TransportVersion} than this value. This includes remote-cluster communication, and communication + * with nodes that are just starting up or otherwise are attempting to join this cluster. + */ public TransportVersion getMinTransportVersion() { return this.minVersions.transportVersion(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java index 36d1947af73ba..cdf01099ee0d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java @@ -74,8 +74,6 @@ default String describeTasks(List tasks) { Strings.collectionToDelimitedStringWithLimit( (Iterable) () -> tasks.stream().map(Object::toString).filter(s -> s.isEmpty() == false).iterator(), ", ", - "", - "", 1024, output ); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 162d90a01720b..2ae42d42cee62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -170,7 +170,9 @@ public DataStream( this.name = name; this.generation = generation; this.metadata = metadata; - assert system == false || hidden; // system indices must be hidden + // The following assert is commented out, because system data streams created before 8.1 are not hidden, + // but should be updated to hidden by 8.18/8.19 (SystemIndexMetadataUpgradeService) + // assert system == false || hidden; // system indices must be hidden this.hidden = hidden; this.replicated = replicated; this.timeProvider = timeProvider; @@ -301,6 +303,13 @@ public boolean isFailureStoreIndex(String indexName) { return failureIndices.containsIndex(indexName); } + /** + * Returns true if the index name provided belongs to this data stream. + */ + public boolean containsIndex(String indexName) { + return backingIndices.containsIndex(indexName) || failureIndices.containsIndex(indexName); + } + public DataStreamOptions getDataStreamOptions() { return dataStreamOptions; } @@ -782,8 +791,9 @@ public DataStream addBackingIndex(Metadata clusterMetadata, Index index) { // ensure that no aliases reference index ensureNoAliasesOnIndex(clusterMetadata, index); - List backingIndices = new ArrayList<>(this.backingIndices.indices); - backingIndices.add(0, index); + List backingIndices = new ArrayList<>(this.backingIndices.indices.size() + 1); + backingIndices.add(index); + backingIndices.addAll(this.backingIndices.indices); assert backingIndices.size() == this.backingIndices.indices.size() + 1; return copy().setBackingIndices(this.backingIndices.copy().setIndices(backingIndices).build()) .setGeneration(generation + 1) @@ -808,8 +818,9 @@ public DataStream addFailureStoreIndex(Metadata clusterMetadata, Index index) { ensureNoAliasesOnIndex(clusterMetadata, index); - List updatedFailureIndices = new ArrayList<>(failureIndices.indices); - updatedFailureIndices.add(0, index); + List updatedFailureIndices = new ArrayList<>(failureIndices.indices.size() + 1); + updatedFailureIndices.add(index); + updatedFailureIndices.addAll(failureIndices.indices); assert updatedFailureIndices.size() == failureIndices.indices.size() + 1; return copy().setFailureIndices(failureIndices.copy().setIndices(updatedFailureIndices).build()) .setGeneration(generation + 1) @@ -1039,7 +1050,7 @@ private boolean isIndexOlderThan( * we return false. */ public boolean isIndexManagedByDataStreamLifecycle(Index index, Function indexMetadataSupplier) { - if (backingIndices.containsIndex(index.getName()) == false && failureIndices.containsIndex(index.getName()) == false) { + if (containsIndex(index.getName()) == false) { return false; } IndexMetadata indexMetadata = indexMetadataSupplier.apply(index.getName()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index 015c90ebe450e..258f12dc7ea7d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; @@ -81,8 +82,7 @@ && isIndexVisible( indexNameExpressionResolver, includeDataStreams )) { - // Resolve any ::* suffixes on the expression. We need to resolve them all to their final valid selectors - resolveSelectorsAndCombine(authorizedIndex, selectorString, indicesOptions, resolvedIndices, metadata); + resolveSelectorsAndCollect(authorizedIndex, selectorString, indicesOptions, resolvedIndices, metadata); } } if (resolvedIndices.isEmpty()) { @@ -98,9 +98,8 @@ && isIndexVisible( } } } else { - // Resolve any ::* suffixes on the expression. We need to resolve them all to their final valid selectors Set resolvedIndices = new HashSet<>(); - resolveSelectorsAndCombine(indexAbstraction, selectorString, indicesOptions, resolvedIndices, metadata); + resolveSelectorsAndCollect(indexAbstraction, selectorString, indicesOptions, resolvedIndices, metadata); if (minus) { finalIndices.removeAll(resolvedIndices); } else if (indicesOptions.ignoreUnavailable() == false || isAuthorized.test(indexAbstraction)) { @@ -114,7 +113,7 @@ && isIndexVisible( return finalIndices; } - private static void resolveSelectorsAndCombine( + private static void resolveSelectorsAndCollect( String indexAbstraction, String selectorString, IndicesOptions indicesOptions, @@ -132,19 +131,8 @@ private static void resolveSelectorsAndCombine( selectorString = IndexComponentSelector.DATA.getKey(); } - if (Regex.isMatchAllPattern(selectorString)) { - // Always accept data - collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, IndexComponentSelector.DATA.getKey())); - // Only put failures on the expression if the abstraction supports it. - if (acceptsAllSelectors) { - collect.add( - IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, IndexComponentSelector.FAILURES.getKey()) - ); - } - } else { - // A non-wildcard selector is always passed along as-is, it's validity for this kind of abstraction is tested later - collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, selectorString)); - } + // A selector is always passed along as-is, it's validity for this kind of abstraction is tested later + collect.add(IndexNameExpressionResolver.combineSelectorExpression(indexAbstraction, selectorString)); } else { assert selectorString == null : "A selector string [" + selectorString + "] is present but selectors are disabled in this context"; @@ -169,8 +157,35 @@ public static boolean isIndexVisible( boolean isVisible = isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index); if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { // it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed. - // complicated to support those options with aliases pointing to multiple indices... + // it's complicated to support those options with aliases pointing to multiple indices... isVisible = isVisible && indicesOptions.ignoreAliases() == false; + + if (isVisible && indexAbstraction.isSystem()) { + // check if it is net new + if (resolver.getNetNewSystemIndexPredicate().test(indexAbstraction.getName())) { + // don't give this code any particular credit for being *correct*. it's just trying to resolve a combination of + // issues in a way that happens to *work*. there's probably a better way of writing things such that this won't + // be necessary, but for the moment, it happens to be expedient to write things this way. + + // unwrap the alias and re-run the function on the write index of the alias -- that is, the alias is visible if + // the concrete index that it refers to is visible + Index writeIndex = indexAbstraction.getWriteIndex(); + if (writeIndex == null) { + return false; + } else { + return isIndexVisible( + expression, + selectorString, + writeIndex.getName(), + indicesOptions, + metadata, + resolver, + includeDataStreams + ); + } + } + } + if (isVisible && selectorString != null) { // Check if a selector was present, and if it is, check if this alias is applicable to it IndexComponentSelector selector = IndexComponentSelector.getByKey(selectorString); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index ae4b16b08862f..33dd20c50e1d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2122,6 +2122,12 @@ public Builder putRolloverInfo(RolloverInfo rolloverInfo) { return this; } + public Builder putRolloverInfos(Map rolloverInfos) { + this.rolloverInfos.clear(); + this.rolloverInfos.putAllFromMap(rolloverInfos); + return this; + } + public long version() { return this.version; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index cb074b1437040..d28049f2a6316 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -364,21 +364,9 @@ protected static Collection resolveExpressionsToResources(Co } } else { if (isExclusion) { - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - resources.remove(new ResolvedExpression(baseExpression, IndexComponentSelector.DATA)); - resources.remove(new ResolvedExpression(baseExpression, IndexComponentSelector.FAILURES)); - } else { - resources.remove(new ResolvedExpression(baseExpression, selector)); - } + resources.remove(new ResolvedExpression(baseExpression, selector)); } else if (ensureAliasOrIndexExists(context, baseExpression, selector)) { - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - resources.add(new ResolvedExpression(baseExpression, IndexComponentSelector.DATA)); - if (context.getState().getMetadata().getIndicesLookup().get(baseExpression).isDataStreamRelated()) { - resources.add(new ResolvedExpression(baseExpression, IndexComponentSelector.FAILURES)); - } - } else { - resources.add(new ResolvedExpression(baseExpression, selector)); - } + resources.add(new ResolvedExpression(baseExpression, selector)); } } } @@ -1046,8 +1034,7 @@ public String[] indexAliases( private static boolean resolvedExpressionsContainsAbstraction(Set resolvedExpressions, String abstractionName) { return resolvedExpressions.contains(new ResolvedExpression(abstractionName)) - || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA)) - || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.ALL_APPLICABLE)); + || resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA)); } /** @@ -1342,8 +1329,7 @@ private static boolean ensureAliasOrIndexExists(Context context, String name, In if (context.options.allowSelectors()) { // Ensure that the selectors are present and that they are compatible with the abstractions they are used with assert selector != null : "Earlier logic should have parsed selectors or added the default selectors already"; - // Check if ::failures has been explicitly requested, since requesting ::* for non-data-stream abstractions would just - // return their data components. + // Check if ::failures has been explicitly requested if (IndexComponentSelector.FAILURES.equals(selector) && indexAbstraction.isDataStreamRelated() == false) { // If requested abstraction is not data stream related, then you cannot use ::failures if (ignoreUnavailable) { @@ -1700,9 +1686,9 @@ private static Set expandToOpenClosed( final IndexMetadata.State excludeState = excludeState(context.getOptions()); Set resources = new HashSet<>(); if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) { - expandToApplicableSelectors(indexAbstraction, selector, resources); + resources.add(new ResolvedExpression(indexAbstraction.getName(), selector)); } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) { - expandToApplicableSelectors(indexAbstraction, selector, resources); + resources.add(new ResolvedExpression(indexAbstraction.getName(), selector)); } else { if (shouldIncludeRegularIndices(context.getOptions(), selector)) { for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) { @@ -1729,31 +1715,6 @@ private static Set expandToOpenClosed( return resources; } - /** - * Adds the abstraction and selector to the results when preserving data streams and aliases at wildcard resolution. If a selector - * is provided, the result is only added if the selector is applicable to the abstraction provided. If - * {@link IndexComponentSelector#ALL_APPLICABLE} is given, the selectors are expanded only to those which are applicable to the - * provided abstraction. - * @param indexAbstraction abstraction to add - * @param selector The selector to add - * @param resources Result collector which is updated with all applicable resolved expressions for a given abstraction and selector - * pair. - */ - private static void expandToApplicableSelectors( - IndexAbstraction indexAbstraction, - IndexComponentSelector selector, - Set resources - ) { - if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { - resources.add(new ResolvedExpression(indexAbstraction.getName(), IndexComponentSelector.DATA)); - if (indexAbstraction.isDataStreamRelated()) { - resources.add(new ResolvedExpression(indexAbstraction.getName(), IndexComponentSelector.FAILURES)); - } - } else if (selector == null || indexAbstraction.isDataStreamRelated() || selector.shouldIncludeFailures() == false) { - resources.add(new ResolvedExpression(indexAbstraction.getName(), selector)); - } - } - private static List resolveEmptyOrTrivialWildcard(Context context, IndexComponentSelector selector) { final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices( context.getOptions(), @@ -2150,20 +2111,10 @@ private static V splitSelectorExpression(String expression, BiFunction getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { - List validationErrors = validateIndexCustomPath(settings, env.sharedDataFile()); + List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 71be34db9626f..26cccef8fd3ea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -30,16 +31,23 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.snapshots.SnapshotInProgressException; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; /** * Handles data stream modification requests. */ public class MetadataDataStreamsService { - + private static final Logger LOGGER = LogManager.getLogger(MetadataDataStreamsService.class); private final ClusterService clusterService; private final IndicesService indicesService; private final DataStreamGlobalRetentionSettings globalRetentionSettings; @@ -343,6 +351,7 @@ private static void addBackingIndex( mapperSupplier, false, failureStore, + dataStream.isSystem(), nodeSettings ); } catch (IOException e) { @@ -410,6 +419,52 @@ private static IndexAbstraction validateIndex(Metadata metadata, String indexNam return index; } + /** + * Removes the given data stream and their backing indices from the Project State. + * + * @param currentState The current cluster state + * @param dataStreams The data streams to remove + * @param settings The settings + * @return The updated Project State + */ + public static ClusterState deleteDataStreams(ClusterState currentState, Set dataStreams, Settings settings) { + if (dataStreams.isEmpty()) { + return currentState; + } + + Set dataStreamNames = dataStreams.stream().map(DataStream::getName).collect(Collectors.toSet()); + Set snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(currentState, dataStreamNames); + if (snapshottingDataStreams.isEmpty() == false) { + throw new SnapshotInProgressException( + "Cannot delete data streams that are being snapshotted: [" + + String.join(", ", snapshottingDataStreams) + + "]. Try again after snapshot finishes or cancel the currently running snapshot." + ); + } + + Set backingIndicesToRemove = new HashSet<>(); + for (DataStream dataStream : dataStreams) { + assert dataStream != null; + if (currentState.metadata().dataStreams().get(dataStream.getName()) == null) { + throw new ResourceNotFoundException("data stream [" + dataStream.getName() + "] not found"); + } + backingIndicesToRemove.addAll(dataStream.getIndices()); + backingIndicesToRemove.addAll(dataStream.getFailureIndices()); + } + + // first delete the data streams and then the indices: + // (this to avoid data stream validation from failing when deleting an index that is part of a data stream + // without updating the data stream) + // TODO: change order when "delete index api" also updates the data stream the "index to be removed" is a member of + Metadata.Builder metadata = Metadata.builder(currentState.metadata()); + for (DataStream ds : dataStreams) { + LOGGER.info("removing data stream [{}]", ds.getName()); + metadata.removeDataStream(ds.getName()); + } + currentState = ClusterState.builder(currentState).metadata(metadata).build(); + return MetadataDeleteIndexService.deleteIndices(currentState, backingIndicesToRemove, settings); + } + /** * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 368270a8bc523..26b598215b1ec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -1167,8 +1167,6 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre Strings.collectionToDelimitedStringWithLimit( indicesToOpen.stream().map(i -> (CharSequence) i.getIndex().toString()).toList(), ",", - "", - "", 512, indexNames ); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 7f8b87d2d3f48..cc5be49d7fe63 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1858,7 +1858,7 @@ private static void validateTemplate(Settings validateSettings, CompressedXConte createdIndex = dummyIndexService.index(); if (mappings != null) { - dummyIndexService.mapperService().merge(MapperService.SINGLE_MAPPING_NAME, mappings, MergeReason.MAPPING_UPDATE); + dummyIndexService.mapperService().merge(MapperService.SINGLE_MAPPING_NAME, mappings, MergeReason.INDEX_TEMPLATE); } } finally { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 5d45bf1ce127e..8b8c3f12cdf9f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -56,7 +56,7 @@ public class MetadataMappingService { public MetadataMappingService(ClusterService clusterService, IndicesService indicesService) { this.clusterService = clusterService; this.indicesService = indicesService; - taskQueue = clusterService.createTaskQueue("put-mapping", Priority.HIGH, new PutMappingExecutor()); + this.taskQueue = clusterService.createTaskQueue("put-mapping", Priority.HIGH, new PutMappingExecutor()); } record PutMappingClusterStateUpdateTask(PutMappingClusterStateUpdateRequest request, ActionListener listener) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 39acc6d3f6311..abae3ab2e3001 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -205,7 +205,7 @@ static void prepareBackingIndex( Function mapperSupplier, boolean removeAlias ) throws IOException { - prepareBackingIndex(b, im, dataStreamName, mapperSupplier, removeAlias, false, Settings.EMPTY); + prepareBackingIndex(b, im, dataStreamName, mapperSupplier, removeAlias, false, false, Settings.EMPTY); } /** @@ -219,6 +219,8 @@ static void prepareBackingIndex( * exception should be thrown in that case instead * @param failureStore true if the index is being migrated into the data stream's failure store, false if it * is being migrated into the data stream's backing indices + * @param makeSystem true if the index is being migrated into the system data stream, false if it + * is being migrated into non-system data stream * @param nodeSettings The settings for the current node */ static void prepareBackingIndex( @@ -228,6 +230,7 @@ static void prepareBackingIndex( Function mapperSupplier, boolean removeAlias, boolean failureStore, + boolean makeSystem, Settings nodeSettings ) throws IOException { MappingMetadata mm = im.mapping(); @@ -258,6 +261,7 @@ static void prepareBackingIndex( imb.mappingVersion(im.getMappingVersion() + 1) .mappingsUpdatedVersion(IndexVersion.current()) .putMapping(new MappingMetadata(mapper)); + imb.system(makeSystem); b.put(imb); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java index ddc6037777b41..4c80e690b07a5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -219,7 +219,7 @@ ClusterState execute(ClusterState currentState) { allocationService.getShardRoutingRoleStrategy(), currentState.routingTable() ); - for (Index index : openIndices) { + for (Index index : new HashSet<>(openIndices)) { // We only want to take on the expense of reopening all shards for an index if the setting is really changing Settings existingSettings = currentState.getMetadata().index(index).getSettings(); boolean needToReopenIndex = false; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java index 228bb3b222a57..71418ee040d93 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java @@ -14,18 +14,28 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemIndexMappingUpdateService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A service responsible for updating the metadata used by system indices. @@ -38,48 +48,62 @@ public class SystemIndexMetadataUpgradeService implements ClusterStateListener { private final SystemIndices systemIndices; private final ClusterService clusterService; - - private volatile boolean updateTaskPending = false; - - private volatile long triggeredVersion = -1L; + private final MasterServiceTaskQueue taskQueue; public SystemIndexMetadataUpgradeService(SystemIndices systemIndices, ClusterService clusterService) { this.systemIndices = systemIndices; this.clusterService = clusterService; + this.taskQueue = clusterService.createTaskQueue( + "system-indices-metadata-upgrade", + Priority.NORMAL, + new SystemIndexMetadataUpgradeExecutor() + ); } @Override public void clusterChanged(ClusterChangedEvent event) { - if (updateTaskPending == false - && event.localNodeMaster() + Metadata currentMetadata = event.state().metadata(); + Metadata previousMetadata = event.previousState().metadata(); + if (event.localNodeMaster() && (event.previousState().nodes().isLocalNodeElectedMaster() == false - || event.state().metadata().indices() != event.previousState().metadata().indices())) { - final Map indexMetadataMap = event.state().metadata().indices(); - final var previousIndices = event.previousState().metadata().indices(); - final long triggerV = event.state().version(); - triggeredVersion = triggerV; + || currentMetadata.indices() != previousMetadata.indices() + || currentMetadata.dataStreams() != previousMetadata.dataStreams())) { + final Map indexMetadataMap = currentMetadata.indices(); + final var previousIndices = previousMetadata.indices(); + Map dataStreams = currentMetadata.dataStreams(); + Map previousDataStreams = previousMetadata.dataStreams(); // Fork to the management pool to avoid blocking the cluster applier thread unnecessarily for very large index counts // TODO: we should have a more efficient way of getting just the changed indices so that we don't have to fork here clusterService.threadPool().executor(ThreadPool.Names.MANAGEMENT).execute(new AbstractRunnable() { @Override protected void doRun() { - if (triggeredVersion != triggerV) { - // don't run if another newer check task was triggered already - return; + Collection changedDataStreams = new ArrayList<>(); + Set dataStreamIndices = new HashSet<>(); + for (Map.Entry cursor : dataStreams.entrySet()) { + DataStream dataStream = cursor.getValue(); + if (dataStream != previousDataStreams.get(cursor.getKey())) { + if (requiresUpdate(dataStream)) { + changedDataStreams.add(dataStream); + } + } + + getIndicesBackingDataStream(dataStream).forEach(dataStreamIndices::add); } + + Collection changedIndices = new ArrayList<>(); for (Map.Entry cursor : indexMetadataMap.entrySet()) { - if (cursor.getValue() != previousIndices.get(cursor.getKey())) { - IndexMetadata indexMetadata = cursor.getValue(); + IndexMetadata indexMetadata = cursor.getValue(); + Index index = indexMetadata.getIndex(); + if (cursor.getValue() != previousIndices.get(cursor.getKey()) && dataStreamIndices.contains(index) == false) { if (requiresUpdate(indexMetadata)) { - updateTaskPending = true; - submitUnbatchedTask( - "system_index_metadata_upgrade_service {system metadata change}", - new SystemIndexMetadataUpdateTask() - ); - break; + changedIndices.add(index); } } } + + if (changedIndices.isEmpty() == false || changedDataStreams.isEmpty() == false) { + submitUpdateTask(changedIndices, changedDataStreams); + } } @Override @@ -91,6 +115,12 @@ public void onFailure(Exception e) { } } + // visible for testing + void submitUpdateTask(Collection changedIndices, Collection changedDataStreams) { + SystemIndexMetadataUpgradeTask task = new SystemIndexMetadataUpgradeTask(changedIndices, changedDataStreams); + taskQueue.submitTask("system-index-metadata-upgrade-service", task, null); + } + // package-private for testing boolean requiresUpdate(IndexMetadata indexMetadata) { final boolean shouldBeSystem = shouldBeSystem(indexMetadata); @@ -107,6 +137,30 @@ boolean requiresUpdate(IndexMetadata indexMetadata) { return false; } + // package-private for testing + boolean requiresUpdate(DataStream dataStream) { + final boolean shouldBeSystem = shouldBeSystem(dataStream); + + // should toggle system index status + if (shouldBeSystem != dataStream.isSystem()) { + return true; + } + + if (shouldBeSystem) { + return dataStream.isHidden() == false; + } + + return false; + } + + private boolean shouldBeSystem(DataStream dataStream) { + return systemIndices.isSystemDataStream(dataStream.getName()); + } + + private static Stream getIndicesBackingDataStream(DataStream dataStream) { + return Stream.concat(dataStream.getIndices().stream(), dataStream.getFailureIndices().stream()); + } + // package-private for testing static boolean isVisible(IndexMetadata indexMetadata) { return indexMetadata.getSettings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false) == false; @@ -114,8 +168,7 @@ static boolean isVisible(IndexMetadata indexMetadata) { // package-private for testing boolean shouldBeSystem(IndexMetadata indexMetadata) { - return systemIndices.isSystemIndex(indexMetadata.getIndex()) - || systemIndices.isSystemIndexBackingDataStream(indexMetadata.getIndex().getName()); + return systemIndices.isSystemIndex(indexMetadata.getIndex()); } // package-private for testing @@ -123,73 +176,152 @@ static boolean hasVisibleAlias(IndexMetadata indexMetadata) { return indexMetadata.getAliases().values().stream().anyMatch(a -> Boolean.FALSE.equals(a.isHidden())); } - @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here - private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { - clusterService.submitUnbatchedStateUpdateTask(source, task); - } + private record SystemIndexMetadataUpgradeTask(Collection changedIndices, Collection changedDataStreams) + implements + ClusterStateTaskListener { - // visible for testing - SystemIndexMetadataUpdateTask getTask() { - return new SystemIndexMetadataUpdateTask(); - } + @Override + public void onFailure(Exception e) { + logger.error("System index metadata upgrade failed", e); + } - public class SystemIndexMetadataUpdateTask extends ClusterStateUpdateTask { + @Override + public String toString() { + return "SystemIndexMetadataUpgradeTask[changedIndices=" + + changedIndices.stream().map(Index::getName).collect(Collectors.joining(",")) + + ";changedDataStreams=" + + changedDataStreams.stream().map(DataStream::getName).collect(Collectors.joining(",")) + + "]"; + } + } + private class SystemIndexMetadataUpgradeExecutor implements ClusterStateTaskExecutor { @Override - public ClusterState execute(ClusterState currentState) throws Exception { - final Map indexMetadataMap = currentState.metadata().indices(); + public ClusterState execute(BatchExecutionContext batchExecutionContext) { + ClusterState initialState = batchExecutionContext.initialState(); + + List> taskContexts = batchExecutionContext.taskContexts(); + List indices = taskContexts.stream() + .map(TaskContext::getTask) + .map(SystemIndexMetadataUpgradeTask::changedIndices) + .flatMap(Collection::stream) + .toList(); + List updatedMetadata = updateIndices(initialState, indices); + + List dataStreams = taskContexts.stream() + .map(TaskContext::getTask) + .map(SystemIndexMetadataUpgradeTask::changedDataStreams) + .flatMap(Collection::stream) + .toList(); + List updatedDataStreams = updateDataStreams(dataStreams); + List updatedBackingIndices = updateIndicesBackingDataStreams(initialState, updatedDataStreams); + + for (TaskContext taskContext : taskContexts) { + taskContext.success(() -> {}); + } + + if (updatedMetadata.isEmpty() == false || updatedDataStreams.isEmpty() == false) { + Metadata.Builder builder = Metadata.builder(initialState.metadata()); + updatedMetadata.forEach(idxMeta -> builder.put(idxMeta, true)); + updatedDataStreams.forEach(builder::put); + updatedBackingIndices.forEach(idxMeta -> builder.put(idxMeta, true)); + + return ClusterState.builder(initialState).metadata(builder).build(); + } + return initialState; + } + + private List updateIndices(ClusterState currentState, List indices) { + if (indices.isEmpty()) { + return Collections.emptyList(); + } + Metadata metadata = currentState.metadata(); final List updatedMetadata = new ArrayList<>(); - for (Map.Entry entry : indexMetadataMap.entrySet()) { - final IndexMetadata indexMetadata = entry.getValue(); - final boolean shouldBeSystem = shouldBeSystem(indexMetadata); - IndexMetadata.Builder builder = IndexMetadata.builder(indexMetadata); - boolean updated = false; - if (shouldBeSystem != indexMetadata.isSystem()) { - builder.system(indexMetadata.isSystem() == false); - updated = true; + for (Index index : indices) { + IndexMetadata indexMetadata = metadata.index(index); + // this might happen because update is async and the index might have been deleted between task creation and execution + if (indexMetadata == null) { + continue; } - if (shouldBeSystem && isVisible(indexMetadata)) { - builder.settings(Settings.builder().put(indexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_HIDDEN, true)); - builder.settingsVersion(builder.settingsVersion() + 1); - updated = true; + final boolean shouldBeSystem = shouldBeSystem(indexMetadata); + IndexMetadata updatedIndexMetadata = updateIndexIfNecessary(indexMetadata, shouldBeSystem); + if (updatedIndexMetadata != null) { + updatedMetadata.add(updatedIndexMetadata); } - if (shouldBeSystem && hasVisibleAlias(indexMetadata)) { - for (AliasMetadata aliasMetadata : indexMetadata.getAliases().values()) { - if (Boolean.FALSE.equals(aliasMetadata.isHidden())) { - builder.removeAlias(aliasMetadata.alias()); - builder.putAlias( - AliasMetadata.builder(aliasMetadata.alias()) - .filter(aliasMetadata.filter()) - .indexRouting(aliasMetadata.indexRouting()) - .isHidden(true) - .searchRouting(aliasMetadata.searchRouting()) - .writeIndex(aliasMetadata.writeIndex()) - ); - } + } + return updatedMetadata; + } + + private IndexMetadata updateIndexIfNecessary(IndexMetadata indexMetadata, boolean shouldBeSystem) { + IndexMetadata.Builder builder = IndexMetadata.builder(indexMetadata); + boolean updated = false; + if (shouldBeSystem != indexMetadata.isSystem()) { + builder.system(indexMetadata.isSystem() == false); + updated = true; + } + if (shouldBeSystem && isVisible(indexMetadata)) { + builder.settings(Settings.builder().put(indexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_HIDDEN, true)); + builder.settingsVersion(builder.settingsVersion() + 1); + updated = true; + } + if (shouldBeSystem && hasVisibleAlias(indexMetadata)) { + for (AliasMetadata aliasMetadata : indexMetadata.getAliases().values()) { + if (Boolean.FALSE.equals(aliasMetadata.isHidden())) { + builder.removeAlias(aliasMetadata.alias()); + builder.putAlias( + AliasMetadata.builder(aliasMetadata.alias()) + .filter(aliasMetadata.filter()) + .indexRouting(aliasMetadata.indexRouting()) + .isHidden(true) + .searchRouting(aliasMetadata.searchRouting()) + .writeIndex(aliasMetadata.writeIndex()) + ); + updated = true; } } - if (updated) { - updatedMetadata.add(builder.build()); - } } + return updated ? builder.build() : null; + } - if (updatedMetadata.isEmpty() == false) { - final Metadata.Builder builder = Metadata.builder(currentState.metadata()); - updatedMetadata.forEach(idxMeta -> builder.put(idxMeta, true)); - return ClusterState.builder(currentState).metadata(builder).build(); + private List updateDataStreams(List dataStreams) { + if (dataStreams.isEmpty()) { + return Collections.emptyList(); + } + List updatedDataStreams = new ArrayList<>(); + for (DataStream dataStream : dataStreams) { + boolean shouldBeSystem = shouldBeSystem(dataStream); + if (dataStream.isSystem() != shouldBeSystem) { + DataStream.Builder dataStreamBuilder = dataStream.copy().setSystem(shouldBeSystem); + if (shouldBeSystem) { + dataStreamBuilder.setHidden(true); + } + + updatedDataStreams.add(dataStreamBuilder.build()); + } } - return currentState; + return updatedDataStreams; } - @Override - public void onFailure(Exception e) { - updateTaskPending = false; - logger.error("failed to update system index metadata", e); + private List updateIndicesBackingDataStreams(ClusterState currentState, List updatedDataStreams) { + if (updatedDataStreams.isEmpty()) { + return Collections.emptyList(); + } + Metadata metadata = currentState.metadata(); + final List updatedMetadata = new ArrayList<>(); + + for (DataStream updatedDataStream : updatedDataStreams) { + boolean shouldBeSystem = updatedDataStream.isSystem(); + List updatedIndicesMetadata = getIndicesBackingDataStreamMetadata(metadata, updatedDataStream).map( + idx -> updateIndexIfNecessary(idx, shouldBeSystem) + ).filter(Objects::nonNull).toList(); + + updatedMetadata.addAll(updatedIndicesMetadata); + } + return updatedMetadata; } - @Override - public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { - updateTaskPending = false; + private Stream getIndicesBackingDataStreamMetadata(Metadata metadata, DataStream dataStream) { + return getIndicesBackingDataStream(dataStream).map(metadata::index); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java index 3651f560e6dde..f64a63332a371 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.transport.Transports; import java.util.Map; @@ -41,6 +42,10 @@ public AllocationStatsService( } public Map stats() { + assert Transports.assertNotTransportThread("too expensive for a transport worker"); + + writeLoadForecaster.refreshLicense(); + var state = clusterService.state(); var info = clusterInfoService.getClusterInfo(); var desiredBalance = desiredBalanceShardsAllocator != null ? desiredBalanceShardsAllocator.getDesiredBalance() : null; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java index e7ca51eee815e..7bebedd9fdde4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java @@ -21,6 +21,8 @@ public interface WriteLoadForecaster { OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata); + void refreshLicense(); + class DefaultWriteLoadForecaster implements WriteLoadForecaster { @Override public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { @@ -31,5 +33,8 @@ public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return OptionalDouble.empty(); } + + @Override + public void refreshLicense() {} } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 27087992f9d2b..e58b1ecb73372 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -170,6 +170,11 @@ private static float ensureValidThreshold(float threshold) { @Override public void allocate(RoutingAllocation allocation) { + if (allocation.metadata().indices().isEmpty() == false) { + // must not use licensed features when just starting up + writeLoadForecaster.refreshLicense(); + } + assert allocation.ignoreDisable() == false; if (allocation.routingNodes().size() == 0) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java index 290e922808b84..6b0f780a886a1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java @@ -1195,7 +1195,7 @@ static List getRestoreFromSnapshotAffectedResources( .collect( toMap( SystemIndices.Feature::getName, - feature -> feature.getIndexDescriptors() + feature -> feature.getSystemResourceDescriptors() .stream() .flatMap(descriptor -> descriptor.getMatchingIndices(metadata).stream()) .collect(toSet()) @@ -1211,29 +1211,6 @@ static List getRestoreFromSnapshotAffectedResources( } } - Map> featureToDsBackingIndices = systemIndices.getFeatures() - .stream() - .collect( - toMap( - SystemIndices.Feature::getName, - feature -> feature.getDataStreamDescriptors() - .stream() - .flatMap(descriptor -> descriptor.getBackingIndexNames(metadata).stream()) - .collect(toSet()) - ) - ); - - // the shards_availability indicator works with indices so let's remove the feature states data streams backing indices from - // the list of affected indices (the feature state will cover the restore of these indices too) - for (Map.Entry> featureToBackingIndices : featureToDsBackingIndices.entrySet()) { - for (String featureIndex : featureToBackingIndices.getValue()) { - if (restoreFromSnapshotIndices.contains(featureIndex)) { - affectedFeatureStates.add(featureToBackingIndices.getKey()); - affectedIndices.remove(featureIndex); - } - } - } - if (affectedIndices.isEmpty() == false) { affectedResources.add(new Diagnosis.Resource(INDEX, affectedIndices.stream().limit(maxAffectedResourcesCount).toList())); } diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java index f756933567683..30d24e243bd19 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java @@ -1701,7 +1701,7 @@ private String buildTasksDescription(List> tasks) { Strings.collectionToDelimitedStringWithLimit((Iterable) () -> tasksBySource.entrySet().stream().map(entry -> { var tasksDescription = executor.describeTasks(entry.getValue()); return tasksDescription.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasksDescription + "]"; - }).filter(s -> s.isEmpty() == false).iterator(), ", ", "", "", MAX_TASK_DESCRIPTION_CHARS, output); + }).filter(s -> s.isEmpty() == false).iterator(), ", ", MAX_TASK_DESCRIPTION_CHARS, output); if (output.length() > MAX_TASK_DESCRIPTION_CHARS) { output.append(" (").append(tasks.size()).append(" tasks in total)"); } diff --git a/server/src/main/java/org/elasticsearch/common/Strings.java b/server/src/main/java/org/elasticsearch/common/Strings.java index 782ce614616ca..e1481ad66736e 100644 --- a/server/src/main/java/org/elasticsearch/common/Strings.java +++ b/server/src/main/java/org/elasticsearch/common/Strings.java @@ -542,54 +542,43 @@ public static Set commaDelimitedListToSet(String str) { * String. E.g. useful for toString() implementations. * * @param coll the Collection to display - * @param delim the delimiter to use (probably a ",") - * @param prefix the String to start each element with - * @param suffix the String to end each element with + * @param delimiter the delimiter to use (probably a ",") * @return the delimited String */ - public static String collectionToDelimitedString(Iterable coll, String delim, String prefix, String suffix) { + public static String collectionToDelimitedString(Iterable coll, String delimiter) { StringBuilder sb = new StringBuilder(); - collectionToDelimitedString(coll, delim, prefix, suffix, sb); + collectionToDelimitedString(coll, delimiter, sb); return sb.toString(); } - public static void collectionToDelimitedString(Iterable coll, String delim, String prefix, String suffix, StringBuilder sb) { + public static void collectionToDelimitedString(Iterable coll, String delimiter, StringBuilder sb) { Iterator it = coll.iterator(); while (it.hasNext()) { - sb.append(prefix).append(it.next()).append(suffix); + sb.append(it.next()); if (it.hasNext()) { - sb.append(delim); + sb.append(delimiter); } } } /** - * Converts a collection of items to a string like {@link #collectionToDelimitedString(Iterable, String, String, String, StringBuilder)} + * Converts a collection of items to a string like {@link #collectionToDelimitedString(Iterable, String, StringBuilder)} * except that it stops if the string gets too long and just indicates how many items were omitted. * * @param coll the collection of items to display - * @param delim the delimiter to write between the items (usually {@code ","}) - * @param prefix a string to write before each item (usually {@code ""} or {@code "["}) - * @param suffix a string to write after each item (usually {@code ""} or {@code "]"}) + * @param delimiter the delimiter to write between the items (e.g. {@code ","}) * @param appendLimit if this many characters have been appended to the string and there are still items to display then the remaining * items are omitted */ - public static void collectionToDelimitedStringWithLimit( - Iterable coll, - String delim, - String prefix, - String suffix, - int appendLimit, - StringBuilder sb - ) { + public static void collectionToDelimitedStringWithLimit(Iterable coll, String delimiter, int appendLimit, StringBuilder sb) { final Iterator it = coll.iterator(); final long lengthLimit = sb.length() + appendLimit; // long to avoid overflow int count = 0; while (it.hasNext()) { - sb.append(prefix).append(it.next()).append(suffix); + sb.append(it.next()); count += 1; if (it.hasNext()) { - sb.append(delim); + sb.append(delimiter); if (sb.length() > lengthLimit) { int omitted = 0; while (it.hasNext()) { @@ -602,18 +591,6 @@ public static void collectionToDelimitedStringWithLimit( } } - /** - * Convenience method to return a Collection as a delimited (e.g. CSV) - * String. E.g. useful for toString() implementations. - * - * @param coll the Collection to display - * @param delim the delimiter to use (probably a ",") - * @return the delimited String - */ - public static String collectionToDelimitedString(Iterable coll, String delim) { - return collectionToDelimitedString(coll, delim, "", ""); - } - /** * Convenience method to return a Collection as a CSV String. * E.g. useful for toString() implementations. diff --git a/server/src/main/java/org/elasticsearch/common/cli/EnvironmentAwareCommand.java b/server/src/main/java/org/elasticsearch/common/cli/EnvironmentAwareCommand.java index 4aebb0ef0117c..2380463ef4611 100644 --- a/server/src/main/java/org/elasticsearch/common/cli/EnvironmentAwareCommand.java +++ b/server/src/main/java/org/elasticsearch/common/cli/EnvironmentAwareCommand.java @@ -84,13 +84,7 @@ protected Environment createEnv(OptionSet options, ProcessInfo processInfo) thro throw new UserException(ExitCodes.USAGE, "setting [" + kvp.key + "] must not be empty"); } if (settings.containsKey(kvp.key)) { - final String message = String.format( - Locale.ROOT, - "setting [%s] already set, saw [%s] and [%s]", - kvp.key, - settings.get(kvp.key), - kvp.value - ); + final String message = String.format(Locale.ROOT, "setting [%s] set twice via command line -E", kvp.key); throw new UserException(ExitCodes.USAGE, message); } settings.put(kvp.key, kvp.value); @@ -133,18 +127,17 @@ private static void putSystemPropertyIfSettingIsMissing( final Map settings, final String setting, final String key - ) { + ) throws UserException { final String value = sysprops.get(key); if (value != null) { if (settings.containsKey(setting)) { final String message = String.format( Locale.ROOT, - "duplicate setting [%s] found via command-line [%s] and system property [%s]", + "setting [%s] found via command-line -E and system property [%s]", setting, - settings.get(setting), - value + key ); - throw new IllegalArgumentException(message); + throw new UserException(ExitCodes.USAGE, message); } else { settings.put(setting, value); } diff --git a/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java b/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java index a900722397edd..0717a20611ad8 100644 --- a/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java +++ b/server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java @@ -16,16 +16,18 @@ import org.elasticsearch.reservedstate.service.FileChangedListener; import java.io.IOException; +import java.io.InputStream; import java.nio.file.ClosedWatchServiceException; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardWatchEventKinds; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; /** * A skeleton service for watching and reacting to a single file changing on disk @@ -119,20 +121,20 @@ public final boolean watching() { // platform independent way to tell if a file changed // we compare the file modified timestamp, the absolute path (symlinks), and file id on the system final boolean watchedFileChanged(Path path) throws IOException { - if (Files.exists(path) == false) { + if (filesExists(path) == false) { return false; } FileUpdateState previousUpdateState = fileUpdateState; - BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class); + BasicFileAttributes attr = filesReadAttributes(path, BasicFileAttributes.class); fileUpdateState = new FileUpdateState(attr.lastModifiedTime().toMillis(), path.toRealPath().toString(), attr.fileKey()); return (previousUpdateState == null || previousUpdateState.equals(fileUpdateState) == false); } protected final synchronized void startWatcher() { - if (Files.exists(watchedFileDir.getParent()) == false) { + if (filesExists(watchedFileDir.getParent()) == false) { logger.warn("File watcher for [{}] cannot start because grandparent directory does not exist", watchedFile); return; } @@ -147,7 +149,7 @@ protected final synchronized void startWatcher() { try { Path settingsDirPath = watchedFileDir(); this.watchService = settingsDirPath.getParent().getFileSystem().newWatchService(); - if (Files.exists(settingsDirPath)) { + if (filesExists(settingsDirPath)) { settingsDirWatchKey = enableDirectoryWatcher(settingsDirWatchKey, settingsDirPath); } else { logger.debug("watched directory [{}] not found, will watch for its creation...", settingsDirPath); @@ -181,7 +183,7 @@ protected final void watcherThread() { Path path = watchedFile(); - if (Files.exists(path)) { + if (filesExists(path)) { logger.debug("found initial operator settings file [{}], applying...", path); processSettingsOnServiceStartAndNotifyListeners(); } else { @@ -209,7 +211,7 @@ protected final void watcherThread() { * real path of our desired file. We don't actually care what changed, we just re-check ourselves. */ Path settingsPath = watchedFileDir(); - if (Files.exists(settingsPath)) { + if (filesExists(settingsPath)) { try { if (logger.isDebugEnabled()) { key.pollEvents().forEach(e -> logger.debug("{}:{}", e.kind().toString(), e.context().toString())); @@ -332,4 +334,19 @@ long retryDelayMillis(int failedCount) { * class to determine if a file has been changed. */ private record FileUpdateState(long timestamp, String path, Object fileKey) {} + + // the following methods are a workaround to ensure exclusive access for files + // required by child watchers; this is required because we only check the caller's module + // not the entire stack + protected abstract boolean filesExists(Path path); + + protected abstract boolean filesIsDirectory(Path path); + + protected abstract A filesReadAttributes(Path path, Class clazz) throws IOException; + + protected abstract Stream filesList(Path dir) throws IOException; + + protected abstract Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException; + + protected abstract InputStream filesNewInputStream(Path path) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/common/file/MasterNodeFileWatchingService.java b/server/src/main/java/org/elasticsearch/common/file/MasterNodeFileWatchingService.java index c106c90708316..cb94be8dd025e 100644 --- a/server/src/main/java/org/elasticsearch/common/file/MasterNodeFileWatchingService.java +++ b/server/src/main/java/org/elasticsearch/common/file/MasterNodeFileWatchingService.java @@ -19,10 +19,13 @@ import org.elasticsearch.gateway.GatewayService; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Instant; +import java.util.stream.Stream; public abstract class MasterNodeFileWatchingService extends AbstractFileWatchingService implements ClusterStateListener { @@ -41,7 +44,7 @@ protected void doStart() { // We start the file watcher when we know we are master from a cluster state change notification. // We need the additional active flag, since cluster state can change after we've shutdown the service // causing the watcher to start again. - this.active = Files.exists(watchedFileDir().getParent()); + this.active = filesExists(watchedFileDir().getParent()); if (active == false) { // we don't have a config directory, we can't possibly launch the file settings service return; @@ -86,9 +89,9 @@ public final void clusterChanged(ClusterChangedEvent event) { */ private void refreshExistingFileStateIfNeeded(ClusterState clusterState) { if (watching()) { - if (shouldRefreshFileState(clusterState) && Files.exists(watchedFile())) { + if (shouldRefreshFileState(clusterState) && filesExists(watchedFile())) { try { - Files.setLastModifiedTime(watchedFile(), FileTime.from(Instant.now())); + filesSetLastModifiedTime(watchedFile(), FileTime.from(Instant.now())); } catch (IOException e) { logger.warn("encountered I/O error trying to update file settings timestamp", e); } @@ -107,4 +110,37 @@ private void refreshExistingFileStateIfNeeded(ClusterState clusterState) { protected boolean shouldRefreshFileState(ClusterState clusterState) { return false; } + + // the following methods are a workaround to ensure exclusive access for files + // required by child watchers; this is required because we only check the caller's module + // not the entire stack + @Override + protected boolean filesExists(Path path) { + return Files.exists(path); + } + + @Override + protected boolean filesIsDirectory(Path path) { + return Files.isDirectory(path); + } + + @Override + protected A filesReadAttributes(Path path, Class clazz) throws IOException { + return Files.readAttributes(path, clazz); + } + + @Override + protected Stream filesList(Path dir) throws IOException { + return Files.list(dir); + } + + @Override + protected Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException { + return Files.setLastModifiedTime(path, time); + } + + @Override + protected InputStream filesNewInputStream(Path path) throws IOException { + return Files.newInputStream(path); + } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java index 745fe034a804a..e1161f03cb8a7 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java @@ -12,6 +12,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.ReleasableBytesReference; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import java.io.IOException; @@ -231,16 +232,24 @@ private static T deserialize( NamedWriteableRegistry registry, BytesReference serialized ) throws IOException { - try ( - StreamInput in = registry == null - ? new DeduplicateStreamInput(serialized.streamInput(), new DeduplicatorCache()) - : new DeduplicateNamedWriteableAwareStreamInput(serialized.streamInput(), registry, new DeduplicatorCache()) - ) { - in.setTransportVersion(serializedAtVersion); - return reader.read(in); + try (StreamInput in = serialized.streamInput()) { + return reader.read(wrapWithDeduplicatorStreamInput(in, serializedAtVersion, registry)); } } + /** Wraps the provided {@link StreamInput} with another stream that extends {@link Deduplicator} */ + public static StreamInput wrapWithDeduplicatorStreamInput( + StreamInput in, + TransportVersion serializedAtVersion, + @Nullable NamedWriteableRegistry registry + ) { + StreamInput out = registry == null + ? new DeduplicateStreamInput(in, new DeduplicatorCache()) + : new DeduplicateNamedWriteableAwareStreamInput(in, registry, new DeduplicatorCache()); + out.setTransportVersion(serializedAtVersion); + return out; + } + /** An object implementing this interface can deduplicate instance of the provided objects.*/ public interface Deduplicator { T deduplicate(T object); diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java index f6d6c7fd68738..134f7746ba627 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java @@ -127,7 +127,7 @@ public static void configure(final Environment environment, boolean useConsole) StatusLogger.getLogger().removeListener(ERROR_LISTENER); } configureESLogging(); - configure(environment.settings(), environment.configFile(), environment.logsFile(), useConsole); + configure(environment.settings(), environment.configDir(), environment.logsDir(), useConsole); initializeStatics(); // creates a permanent status logger that can watch for StatusLogger events and forward to a real logger configureStatusLoggerForwarder(); diff --git a/server/src/main/java/org/elasticsearch/common/logging/internal/LevelUtil.java b/server/src/main/java/org/elasticsearch/common/logging/internal/LevelUtil.java index df939dd0f2d21..c1d6a80ee4618 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/internal/LevelUtil.java +++ b/server/src/main/java/org/elasticsearch/common/logging/internal/LevelUtil.java @@ -9,20 +9,51 @@ package org.elasticsearch.common.logging.internal; +import static org.apache.logging.log4j.Level.ALL; +import static org.apache.logging.log4j.Level.DEBUG; +import static org.apache.logging.log4j.Level.ERROR; +import static org.apache.logging.log4j.Level.FATAL; +import static org.apache.logging.log4j.Level.INFO; +import static org.apache.logging.log4j.Level.OFF; +import static org.apache.logging.log4j.Level.TRACE; +import static org.apache.logging.log4j.Level.WARN; + public final class LevelUtil { private LevelUtil() {} public static org.apache.logging.log4j.Level log4jLevel(final org.elasticsearch.logging.Level level) { return switch (level) { - case OFF -> org.apache.logging.log4j.Level.OFF; - case FATAL -> org.apache.logging.log4j.Level.FATAL; + case OFF -> OFF; + case FATAL -> FATAL; case ERROR -> org.apache.logging.log4j.Level.ERROR; - case WARN -> org.apache.logging.log4j.Level.WARN; + case WARN -> WARN; case INFO -> org.apache.logging.log4j.Level.INFO; case DEBUG -> org.apache.logging.log4j.Level.DEBUG; - case TRACE -> org.apache.logging.log4j.Level.TRACE; + case TRACE -> TRACE; case ALL -> org.apache.logging.log4j.Level.ALL; }; } + + public static org.elasticsearch.logging.Level elasticsearchLevel(final org.apache.logging.log4j.Level log4jLevel) { + // we can't use a switch because log4j levels are not an enum + if (log4jLevel == OFF) { + return org.elasticsearch.logging.Level.OFF; + } else if (log4jLevel == FATAL) { + return org.elasticsearch.logging.Level.FATAL; + } else if (log4jLevel == ERROR) { + return org.elasticsearch.logging.Level.ERROR; + } else if (log4jLevel == WARN) { + return org.elasticsearch.logging.Level.WARN; + } else if (log4jLevel == INFO) { + return org.elasticsearch.logging.Level.INFO; + } else if (log4jLevel == DEBUG) { + return org.elasticsearch.logging.Level.DEBUG; + } else if (log4jLevel == TRACE) { + return org.elasticsearch.logging.Level.TRACE; + } else if (log4jLevel == ALL) { + return org.elasticsearch.logging.Level.ALL; + } + throw new AssertionError("unknown log4j level [" + log4jLevel + "]"); + } } diff --git a/server/src/main/java/org/elasticsearch/common/logging/internal/LoggerFactoryImpl.java b/server/src/main/java/org/elasticsearch/common/logging/internal/LoggerFactoryImpl.java index 6b92f87a9be23..393a94125da60 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/internal/LoggerFactoryImpl.java +++ b/server/src/main/java/org/elasticsearch/common/logging/internal/LoggerFactoryImpl.java @@ -10,6 +10,8 @@ package org.elasticsearch.common.logging.internal; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.logging.Level; import org.elasticsearch.logging.Logger; import org.elasticsearch.logging.internal.spi.LoggerFactory; @@ -22,6 +24,23 @@ public Logger getLogger(String name) { @Override public Logger getLogger(Class clazz) { - return new LoggerImpl(LogManager.getLogger(clazz)); + // Elasticsearch configures logging at the root level, it does not support + // programmatic configuration at the logger level. Log4j's method for + // getting a logger by Class doesn't just use the class name, but also + // scans the classloader hierarchy for programmatic configuration. Here we + // just delegate to use the String class name so that regardless of which + // classloader a class comes from, we will use the root logging config. + return getLogger(clazz.getName()); + } + + @Override + public void setRootLevel(Level level) { + var log4jLevel = LevelUtil.log4jLevel(level); + Loggers.setLevel(LogManager.getRootLogger(), log4jLevel); + } + + @Override + public Level getRootLevel() { + return LevelUtil.elasticsearchLevel(LogManager.getRootLogger().getLevel()); } } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 5043508c781f0..bd756537a002f 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -20,6 +20,8 @@ import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FilterCodecReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; @@ -189,7 +191,18 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc throw new IllegalStateException("no commit found in the directory"); } } + // Need to figure out what the parent field is that, so that validation in IndexWriter doesn't fail + // if no parent field is configured, but FieldInfo says there is a parent field. + String parentField = null; final IndexCommit cp = getIndexCommit(si, directory); + try (var reader = DirectoryReader.open(cp)) { + var topLevelFieldInfos = FieldInfos.getMergedFieldInfos(reader); + for (FieldInfo fieldInfo : topLevelFieldInfos) { + if (fieldInfo.isParentField()) { + parentField = fieldInfo.getName(); + } + } + } try ( IndexWriter writer = new IndexWriter( directory, @@ -197,6 +210,7 @@ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Direc .setIndexCommit(cp) .setCommitOnClose(false) .setOpenMode(IndexWriterConfig.OpenMode.APPEND) + .setParentField(parentField) ) ) { // do nothing and close this will kick off IndexFileDeleter which will remove all pending files diff --git a/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java b/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java index e4f1608a52d15..bd32df65937db 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java +++ b/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java @@ -142,7 +142,7 @@ public LocallyMountedSecrets(Environment environment) { * @return Secrets directory within an Elasticsearch environment */ public static Path resolveSecretsDir(Environment environment) { - return environment.configFile().toAbsolutePath().resolve(SECRETS_DIRECTORY); + return environment.configDir().toAbsolutePath().resolve(SECRETS_DIRECTORY); } /** diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 2abfee670b950..784fb5ebc63d3 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -290,6 +290,28 @@ public String get(String setting, String defaultValue) { return retVal == null ? defaultValue : retVal; } + /** + * Returns the values for the given settings pattern. + * + * Either a concrete setting name, or a pattern containing a single glob is supported. + * + * @param settingPattern name of a setting or a setting name pattern containing a glob + * @return zero or more values for any settings in this settings object that match the given pattern + */ + public Stream getValues(String settingPattern) { + int globIndex = settingPattern.indexOf(".*."); + Stream settingNames; + if (globIndex == -1) { + settingNames = Stream.of(settingPattern); + } else { + String prefix = settingPattern.substring(0, globIndex + 1); + String suffix = settingPattern.substring(globIndex + 2); + Settings subSettings = getByPrefix(prefix); + settingNames = subSettings.names().stream().map(k -> prefix + k + suffix); + } + return settingNames.map(this::getAsList).flatMap(List::stream).filter(Objects::nonNull); + } + /** * Returns the setting value (as float) associated with the setting key. If it does not exists, * returns the default value provided. diff --git a/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java b/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java index 2df7e6537c609..3aa7c67a14c65 100644 --- a/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java +++ b/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java @@ -30,18 +30,29 @@ public SizeLimitingStringWriter(int sizeLimit) { this.sizeLimit = sizeLimit; } - private void checkSizeLimit(int additionalChars) { - int bufLen = getBuffer().length(); - if (bufLen + additionalChars > sizeLimit) { - throw new SizeLimitExceededException( - Strings.format("String [%s...] has exceeded the size limit [%s]", getBuffer().substring(0, Math.min(bufLen, 20)), sizeLimit) - ); + private int limitSize(int additionalChars) { + int neededSize = getBuffer().length() + additionalChars; + if (neededSize > sizeLimit) { + return additionalChars - (neededSize - sizeLimit); } + return additionalChars; + } + + private void throwSizeLimitExceeded(int limitedChars, int requestedChars) { + assert limitedChars < requestedChars; + int bufLen = getBuffer().length(); + int foundSize = bufLen - limitedChars + requestedChars; // reconstitute original + String selection = getBuffer().substring(0, Math.min(bufLen, 20)); + throw new SizeLimitExceededException( + Strings.format("String [%s...] has size [%d] which exceeds the size limit [%d]", selection, foundSize, sizeLimit) + ); } @Override public void write(int c) { - checkSizeLimit(1); + if (limitSize(1) != 1) { + throwSizeLimitExceeded(0, 1); + } super.write(c); } @@ -49,20 +60,29 @@ public void write(int c) { @Override public void write(char[] cbuf, int off, int len) { - checkSizeLimit(len); - super.write(cbuf, off, len); + int limitedLen = limitSize(len); + if (limitedLen > 0) { + super.write(cbuf, off, limitedLen); + } + if (limitedLen != len) { + throwSizeLimitExceeded(limitedLen, len); + } } @Override public void write(String str) { - checkSizeLimit(str.length()); - super.write(str); + this.write(str, 0, str.length()); } @Override public void write(String str, int off, int len) { - checkSizeLimit(len); - super.write(str, off, len); + int limitedLen = limitSize(len); + if (limitedLen > 0) { + super.write(str, off, limitedLen); + } + if (limitedLen != len) { + throwSizeLimitExceeded(limitedLen, len); + } } // append(...) delegates to write(...) methods diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index 73597fde0cbc9..3d5f2427bac08 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -191,7 +191,7 @@ public static ZoneId of(String zoneId) { /** * convert a java time instant to a long value which is stored in lucene - * the long value resembles the nanoseconds since the epoch + * the long value represents the nanoseconds since the epoch * * @param instant the instant to convert * @return the nano seconds and seconds as a single long @@ -210,10 +210,35 @@ public static long toLong(Instant instant) { return instant.getEpochSecond() * 1_000_000_000 + instant.getNano(); } + /** + * Convert a java time instant to a long value which is stored in lucene, + * the long value represents the milliseconds since epoch + * + * @param instant the instant to convert + * @return the total milliseconds as a single long + */ + public static long toLongMillis(Instant instant) { + try { + return instant.toEpochMilli(); + } catch (ArithmeticException e) { + if (instant.isAfter(Instant.now())) { + throw new IllegalArgumentException( + "date[" + instant + "] is too far in the future to be represented in a long milliseconds variable", + e + ); + } else { + throw new IllegalArgumentException( + "date[" + instant + "] is too far in the past to be represented in a long milliseconds variable", + e + ); + } + } + } + /** * Returns an instant that is with valid nanosecond resolution. If * the parameter is before the valid nanosecond range then this returns - * the minimum {@linkplain Instant} valid for nanosecond resultion. If + * the minimum {@linkplain Instant} valid for nanosecond resolution. If * the parameter is after the valid nanosecond range then this returns * the maximum {@linkplain Instant} valid for nanosecond resolution. *

@@ -423,8 +448,10 @@ public static ZonedDateTime nowWithMillisResolution(Clock clock) { private static final boolean USES_COMPAT = System.getProperty("java.locale.providers", "").contains("COMPAT"); // check for all textual fields, and localized zone offset // the weird thing with Z is to ONLY match 4 in a row, with no Z before or after (but those groups can also be empty) + private static final Predicate LEGACY_DATE_FORMAT_MATCHER = Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(? CONTAINS_CHANGING_TEXT_SPECIFIERS = USES_COMPAT - ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(? CONTAINS_WEEK_DATE_SPECIFIERS = USES_COMPAT @@ -452,4 +479,8 @@ static void checkTextualDateFormats(String format) { ); } } + + public static boolean containsCompatOnlyDateFormat(String format) { + return LEGACY_DATE_FORMAT_MATCHER.test(format); + } } diff --git a/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java b/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java index 86e82886ed263..f334870906d94 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java +++ b/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java @@ -14,15 +14,15 @@ import java.util.MissingResourceException; /** - * Utilities for for dealing with {@link Locale} objects + * Utilities for dealing with {@link Locale} objects */ public class LocaleUtils { /** * Parse the given locale as {@code language}, {@code language-country} or * {@code language-country-variant}. - * Either underscores or hyphens may be used as separators, but consistently, ie. - * you may not use an hyphen to separate the language from the country and an + * Either underscores or hyphens may be used as separators, but consistently, i.e. + * you may not use a hyphen to separate the language from the country and an * underscore to separate the country from the variant. * @throws IllegalArgumentException if there are too many parts in the locale string * @throws IllegalArgumentException if the language or country is not recognized diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index 9120576815bac..d16cf5eb297db 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -96,6 +96,21 @@ public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing( return new PrioritizedEsThreadPoolExecutor(name, 1, 1, 0L, TimeUnit.MILLISECONDS, threadFactory, contextHolder, timer); } + /** + * Creates a scaling {@link EsThreadPoolExecutor} using an unbounded work queue. + *

+ * The {@link EsThreadPoolExecutor} scales the same way as a regular {@link ThreadPoolExecutor} until the core pool size + * (and at least 1) is reached: each time a task is submitted a new worker is added regardless if an idle worker is available. + *

+ * Once having reached the core pool size, a {@link ThreadPoolExecutor} will only add a new worker if the work queue rejects + * a task offer. Typically, using a regular unbounded queue, task offers won't ever be rejected, meaning the worker pool would never + * scale beyond the core pool size. + *

+ * Scaling {@link EsThreadPoolExecutor}s use a customized unbounded {@link LinkedTransferQueue}, which rejects every task offer unless + * it can be immediately transferred to an available idle worker. If no such worker is available, the executor will add + * a new worker if capacity remains, otherwise the task is rejected and then appended to the work queue via the {@link ForceQueuePolicy} + * rejection handler. + */ public static EsThreadPoolExecutor newScaling( String name, int min, @@ -107,10 +122,12 @@ public static EsThreadPoolExecutor newScaling( ThreadContext contextHolder, TaskTrackingConfig config ) { - ExecutorScalingQueue queue = new ExecutorScalingQueue<>(); - EsThreadPoolExecutor executor; + LinkedTransferQueue queue = newUnboundedScalingLTQueue(min, max); + // Force queued work via ForceQueuePolicy might starve if no worker is available (if core size is empty), + // probing the worker pool prevents this. + boolean probeWorkerPool = min == 0 && queue instanceof ExecutorScalingQueue; if (config.trackExecutionTime()) { - executor = new TaskExecutionTimeTrackingEsThreadPoolExecutor( + return new TaskExecutionTimeTrackingEsThreadPoolExecutor( name, min, max, @@ -119,12 +136,12 @@ public static EsThreadPoolExecutor newScaling( queue, TimedRunnable::new, threadFactory, - new ForceQueuePolicy(rejectAfterShutdown), + new ForceQueuePolicy(rejectAfterShutdown, probeWorkerPool), contextHolder, config ); } else { - executor = new EsThreadPoolExecutor( + return new EsThreadPoolExecutor( name, min, max, @@ -132,14 +149,27 @@ public static EsThreadPoolExecutor newScaling( unit, queue, threadFactory, - new ForceQueuePolicy(rejectAfterShutdown), + new ForceQueuePolicy(rejectAfterShutdown, probeWorkerPool), contextHolder ); } - queue.executor = executor; - return executor; } + /** + * Creates a scaling {@link EsThreadPoolExecutor} using an unbounded work queue. + *

+ * The {@link EsThreadPoolExecutor} scales the same way as a regular {@link ThreadPoolExecutor} until the core pool size + * (and at least 1) is reached: each time a task is submitted a new worker is added regardless if an idle worker is available. + *

+ * Once having reached the core pool size, a {@link ThreadPoolExecutor} will only add a new worker if the work queue rejects + * a task offer. Typically, using a regular unbounded queue, task offers won't ever be rejected, meaning the worker pool would never + * scale beyond the core pool size. + *

+ * Scaling {@link EsThreadPoolExecutor}s use a customized unbounded {@link LinkedTransferQueue}, which rejects every task offer unless + * it can be immediately transferred to an available idle worker. If no such worker is available, the executor will add + * a new worker if capacity remains, otherwise the task is rejected and then appended to the work queue via the {@link ForceQueuePolicy} + * rejection handler. + */ public static EsThreadPoolExecutor newScaling( String name, int min, @@ -389,32 +419,58 @@ public boolean isSystem() { */ private EsExecutors() {} - static class ExecutorScalingQueue extends LinkedTransferQueue { + private static LinkedTransferQueue newUnboundedScalingLTQueue(int corePoolSize, int maxPoolSize) { + if (maxPoolSize == 1 || maxPoolSize == corePoolSize) { + // scaling beyond core pool size (or 1) not required, use a regular unbounded LinkedTransferQueue + return new LinkedTransferQueue<>(); + } + // scaling beyond core pool size with an unbounded queue requires ExecutorScalingQueue + // note, reconfiguration of core / max pool size not supported in EsThreadPoolExecutor + return new ExecutorScalingQueue<>(); + } - ThreadPoolExecutor executor; + /** + * Customized {@link LinkedTransferQueue} to allow a {@link ThreadPoolExecutor} to scale beyond its core pool size despite having an + * unbounded queue. + *

+ * Note, usage of unbounded work queues is a problem by itself. For once, it makes error-prone customizations necessary so that + * thread pools can scale up adequately. But worse, infinite queues prevent backpressure and impose a high risk of causing OOM errors. + * Github #18613 captures various long outstanding, but important + * improvements to thread pools. + *

+ * Once having reached its core pool size, a {@link ThreadPoolExecutor} will only add more workers if capacity remains and + * the task offer is rejected by the work queue. Typically that's never the case using a regular unbounded queue. + *

+ * This customized implementation rejects every task offer unless it can be immediately transferred to an available idle worker. + * It relies on {@link ForceQueuePolicy} rejection handler to append the task to the work queue if no additional worker can be added + * and the task is rejected by the executor. + *

+ * Note, {@link ForceQueuePolicy} cannot guarantee there will be available workers when appending tasks directly to the queue. + * For that reason {@link ExecutorScalingQueue} cannot be used with executors with empty core and max pool size of 1: + * the only available worker could time out just about at the same time as the task is appended, see + * Github #124667 for more details. + *

+ * Note, configuring executors using core = max size in combination with {@code allowCoreThreadTimeOut} could be an alternative to + * {@link ExecutorScalingQueue}. However, the scaling behavior would be very different: Using {@link ExecutorScalingQueue} + * we are able to reuse idle workers if available by means of {@link ExecutorScalingQueue#tryTransfer(Object)}. + * If setting core = max size, the executor will add a new worker for every task submitted until reaching the core/max pool size + * even if there's idle workers available. + */ + static class ExecutorScalingQueue extends LinkedTransferQueue { ExecutorScalingQueue() {} @Override public boolean offer(E e) { - // first try to transfer to a waiting worker thread - if (tryTransfer(e) == false) { - // check if there might be spare capacity in the thread - // pool executor - int left = executor.getMaximumPoolSize() - executor.getCorePoolSize(); - if (left > 0) { - // reject queuing the task to force the thread pool - // executor to add a worker if it can; combined - // with ForceQueuePolicy, this causes the thread - // pool to always scale up to max pool size and we - // only queue when there is no spare capacity - return false; - } else { - return super.offer(e); - } - } else { - return true; + if (e == EsThreadPoolExecutor.WORKER_PROBE) { // referential equality + // this probe ensures a worker is available after force queueing a task via ForceQueuePolicy + return super.offer(e); } + // try to transfer to a waiting worker thread + // otherwise reject queuing the task to force the thread pool executor to add a worker if it can; + // combined with ForceQueuePolicy, this causes the thread pool to always scale up to max pool size + // so that we only queue when there is no spare capacity + return tryTransfer(e); } // Overridden to workaround a JDK bug introduced in JDK 21.0.2 @@ -456,15 +512,24 @@ static class ForceQueuePolicy extends EsRejectedExecutionHandler { */ private final boolean rejectAfterShutdown; + /** + * Flag to indicate if the worker pool needs to be probed after force queuing a task to guarantee a worker is available. + */ + private final boolean probeWorkerPool; + /** * @param rejectAfterShutdown indicates if {@link Runnable} should be rejected once the thread pool is shutting down */ - ForceQueuePolicy(boolean rejectAfterShutdown) { + ForceQueuePolicy(boolean rejectAfterShutdown, boolean probeWorkerPool) { this.rejectAfterShutdown = rejectAfterShutdown; + this.probeWorkerPool = probeWorkerPool; } @Override public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) { + if (task == EsThreadPoolExecutor.WORKER_PROBE) { // referential equality + return; + } if (rejectAfterShutdown) { if (executor.isShutdown()) { reject(executor, task); @@ -481,12 +546,19 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) { } } - private static void put(ThreadPoolExecutor executor, Runnable task) { + private void put(ThreadPoolExecutor executor, Runnable task) { final BlockingQueue queue = executor.getQueue(); - // force queue policy should only be used with a scaling queue - assert queue instanceof ExecutorScalingQueue; + // force queue policy should only be used with a scaling queue (ExecutorScalingQueue / LinkedTransferQueue) + assert queue instanceof LinkedTransferQueue; try { queue.put(task); + if (probeWorkerPool && task == queue.peek()) { // referential equality + // If the task is at the head of the queue, we can assume the queue was previously empty. In this case available workers + // might have timed out in the meanwhile. To prevent the task from starving, we submit a noop probe to the executor. + // Note, this deliberately doesn't check getPoolSize()==0 to avoid potential race conditions, + // as the count in the atomic state (used by workerCountOf) is decremented first. + executor.execute(EsThreadPoolExecutor.WORKER_PROBE); + } } catch (final InterruptedException e) { assert false : "a scaling queue never blocks so a put to it can never be interrupted"; throw new AssertionError(e); diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java index a4d2777a48b63..ad4616692850e 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java @@ -29,6 +29,15 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor { private static final Logger logger = LogManager.getLogger(EsThreadPoolExecutor.class); + // noop probe to prevent starvation of work in the work queue due to ForceQueuePolicy + // https://github.com/elastic/elasticsearch/issues/124667 + // note, this is intentionally not a lambda to avoid this ever be turned into a compile time constant + // matching similar lambdas coming from other places + static final Runnable WORKER_PROBE = new Runnable() { + @Override + public void run() {} + }; + private final ThreadContext contextHolder; /** @@ -66,9 +75,19 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor { this.contextHolder = contextHolder; } + @Override + public void setCorePoolSize(int corePoolSize) { + throw new UnsupportedOperationException("reconfiguration at runtime is not supported"); + } + + @Override + public void setMaximumPoolSize(int maximumPoolSize) { + throw new UnsupportedOperationException("reconfiguration at runtime is not supported"); + } + @Override public void execute(Runnable command) { - final Runnable wrappedRunnable = wrapRunnable(command); + final Runnable wrappedRunnable = command != WORKER_PROBE ? wrapRunnable(command) : WORKER_PROBE; try { super.execute(wrappedRunnable); } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/env/Environment.java b/server/src/main/java/org/elasticsearch/env/Environment.java index 39232f2f4ac97..21b63c5c1b89f 100644 --- a/server/src/main/java/org/elasticsearch/env/Environment.java +++ b/server/src/main/java/org/elasticsearch/env/Environment.java @@ -47,28 +47,28 @@ public class Environment { private final Settings settings; - private final Path[] dataFiles; + private final Path[] dataDirs; - private final Path[] repoFiles; + private final Path[] repoDirs; - private final Path configFile; + private final Path configDir; - private final Path pluginsFile; + private final Path pluginsDir; - private final Path modulesFile; + private final Path modulesDir; - private final Path sharedDataFile; + private final Path sharedDataDir; /** location of bin/, used by plugin manager */ - private final Path binFile; + private final Path binDir; /** location of lib/, */ - private final Path libFile; + private final Path libDir; - private final Path logsFile; + private final Path logsDir; /** Path to the temporary file directory used by the JDK */ - private final Path tmpFile; + private final Path tmpDir; public Environment(final Settings settings, final Path configPath) { this(settings, configPath, PathUtils.get(System.getProperty("java.io.tmpdir"))); @@ -84,67 +84,67 @@ public Environment(final Settings settings, final Path configPath) { } if (configPath != null) { - configFile = configPath.toAbsolutePath().normalize(); + configDir = configPath.toAbsolutePath().normalize(); } else { - configFile = homeFile.resolve("config"); + configDir = homeFile.resolve("config"); } - tmpFile = Objects.requireNonNull(tmpPath); + tmpDir = Objects.requireNonNull(tmpPath); - pluginsFile = homeFile.resolve("plugins"); + pluginsDir = homeFile.resolve("plugins"); List dataPaths = PATH_DATA_SETTING.get(settings); if (dataPaths.isEmpty() == false) { - dataFiles = new Path[dataPaths.size()]; + dataDirs = new Path[dataPaths.size()]; for (int i = 0; i < dataPaths.size(); i++) { - dataFiles[i] = PathUtils.get(dataPaths.get(i)).toAbsolutePath().normalize(); + dataDirs[i] = PathUtils.get(dataPaths.get(i)).toAbsolutePath().normalize(); } } else { - dataFiles = new Path[] { homeFile.resolve("data") }; + dataDirs = new Path[] { homeFile.resolve("data") }; } if (PATH_SHARED_DATA_SETTING.exists(settings)) { - sharedDataFile = PathUtils.get(PATH_SHARED_DATA_SETTING.get(settings)).toAbsolutePath().normalize(); + sharedDataDir = PathUtils.get(PATH_SHARED_DATA_SETTING.get(settings)).toAbsolutePath().normalize(); } else { - sharedDataFile = null; + sharedDataDir = null; } List repoPaths = PATH_REPO_SETTING.get(settings); if (repoPaths.isEmpty()) { - repoFiles = EMPTY_PATH_ARRAY; + repoDirs = EMPTY_PATH_ARRAY; } else { - repoFiles = new Path[repoPaths.size()]; + repoDirs = new Path[repoPaths.size()]; for (int i = 0; i < repoPaths.size(); i++) { - repoFiles[i] = PathUtils.get(repoPaths.get(i)).toAbsolutePath().normalize(); + repoDirs[i] = PathUtils.get(repoPaths.get(i)).toAbsolutePath().normalize(); } } // this is trappy, Setting#get(Settings) will get a fallback setting yet return false for Settings#exists(Settings) if (PATH_LOGS_SETTING.exists(settings)) { - logsFile = PathUtils.get(PATH_LOGS_SETTING.get(settings)).toAbsolutePath().normalize(); + logsDir = PathUtils.get(PATH_LOGS_SETTING.get(settings)).toAbsolutePath().normalize(); } else { - logsFile = homeFile.resolve("logs"); + logsDir = homeFile.resolve("logs"); } - binFile = homeFile.resolve("bin"); - libFile = homeFile.resolve("lib"); - modulesFile = homeFile.resolve("modules"); + binDir = homeFile.resolve("bin"); + libDir = homeFile.resolve("lib"); + modulesDir = homeFile.resolve("modules"); final Settings.Builder finalSettings = Settings.builder().put(settings); if (PATH_DATA_SETTING.exists(settings)) { if (dataPathUsesList(settings)) { - finalSettings.putList(PATH_DATA_SETTING.getKey(), Arrays.stream(dataFiles).map(Path::toString).toList()); + finalSettings.putList(PATH_DATA_SETTING.getKey(), Arrays.stream(dataDirs).map(Path::toString).toList()); } else { - assert dataFiles.length == 1; - finalSettings.put(PATH_DATA_SETTING.getKey(), dataFiles[0]); + assert dataDirs.length == 1; + finalSettings.put(PATH_DATA_SETTING.getKey(), dataDirs[0]); } } finalSettings.put(PATH_HOME_SETTING.getKey(), homeFile); - finalSettings.put(PATH_LOGS_SETTING.getKey(), logsFile.toString()); + finalSettings.put(PATH_LOGS_SETTING.getKey(), logsDir.toString()); if (PATH_REPO_SETTING.exists(settings)) { - finalSettings.putList(Environment.PATH_REPO_SETTING.getKey(), Arrays.stream(repoFiles).map(Path::toString).toList()); + finalSettings.putList(Environment.PATH_REPO_SETTING.getKey(), Arrays.stream(repoDirs).map(Path::toString).toList()); } if (PATH_SHARED_DATA_SETTING.exists(settings)) { - assert sharedDataFile != null; - finalSettings.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), sharedDataFile.toString()); + assert sharedDataDir != null; + finalSettings.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), sharedDataDir.toString()); } this.settings = finalSettings.build(); @@ -160,22 +160,22 @@ public Settings settings() { /** * The data location. */ - public Path[] dataFiles() { - return dataFiles; + public Path[] dataDirs() { + return dataDirs; } /** * The shared data location */ - public Path sharedDataFile() { - return sharedDataFile; + public Path sharedDataDir() { + return sharedDataDir; } /** * The shared filesystem repo locations. */ - public Path[] repoFiles() { - return repoFiles; + public Path[] repoDirs() { + return repoDirs; } /** @@ -183,8 +183,8 @@ public Path[] repoFiles() { * * If the specified location doesn't match any of the roots, returns null. */ - public Path resolveRepoFile(String location) { - return PathUtils.get(repoFiles, location); + public Path resolveRepoDir(String location) { + return PathUtils.get(repoDirs, location); } /** @@ -198,7 +198,7 @@ public URL resolveRepoURL(URL url) { if ("file".equalsIgnoreCase(url.getProtocol())) { if (url.getHost() == null || "".equals(url.getHost())) { // only local file urls are supported - Path path = PathUtils.get(repoFiles, url.toURI()); + Path path = PathUtils.get(repoDirs, url.toURI()); if (path == null) { // Couldn't resolve against known repo locations return null; @@ -237,45 +237,45 @@ public URL resolveRepoURL(URL url) { /** * The config directory. */ - public Path configFile() { - return configFile; + public Path configDir() { + return configDir; } - public Path pluginsFile() { - return pluginsFile; + public Path pluginsDir() { + return pluginsDir; } - public Path binFile() { - return binFile; + public Path binDir() { + return binDir; } - public Path libFile() { - return libFile; + public Path libDir() { + return libDir; } - public Path modulesFile() { - return modulesFile; + public Path modulesDir() { + return modulesDir; } - public Path logsFile() { - return logsFile; + public Path logsDir() { + return logsDir; } /** Path to the default temp directory used by the JDK */ - public Path tmpFile() { - return tmpFile; + public Path tmpDir() { + return tmpDir; } /** Ensure the configured temp directory is a valid directory */ - public void validateTmpFile() throws IOException { - validateTemporaryDirectory("Temporary directory", tmpFile); + public void validateTmpDir() throws IOException { + validateTemporaryDirectory("Temporary directory", tmpDir); } /** * Ensure the temp directories needed for JNA are set up correctly. */ public void validateNativesConfig() throws IOException { - validateTmpFile(); + validateTmpDir(); if (Constants.LINUX) { validateTemporaryDirectory(LIBFFI_TMPDIR_ENVIRONMENT_VARIABLE + " environment variable", getLibffiTemporaryDirectory()); } @@ -336,15 +336,15 @@ public static long getUsableSpace(Path path) throws IOException { * object which may contain different setting) */ public static void assertEquivalent(Environment actual, Environment expected) { - assertEquals(actual.dataFiles(), expected.dataFiles(), "dataFiles"); - assertEquals(actual.repoFiles(), expected.repoFiles(), "repoFiles"); - assertEquals(actual.configFile(), expected.configFile(), "configFile"); - assertEquals(actual.pluginsFile(), expected.pluginsFile(), "pluginsFile"); - assertEquals(actual.binFile(), expected.binFile(), "binFile"); - assertEquals(actual.libFile(), expected.libFile(), "libFile"); - assertEquals(actual.modulesFile(), expected.modulesFile(), "modulesFile"); - assertEquals(actual.logsFile(), expected.logsFile(), "logsFile"); - assertEquals(actual.tmpFile(), expected.tmpFile(), "tmpFile"); + assertEquals(actual.dataDirs(), expected.dataDirs(), "dataDirs"); + assertEquals(actual.repoDirs(), expected.repoDirs(), "sharedRepoDirs"); + assertEquals(actual.configDir(), expected.configDir(), "configDir"); + assertEquals(actual.pluginsDir(), expected.pluginsDir(), "pluginsDir"); + assertEquals(actual.binDir(), expected.binDir(), "binDir"); + assertEquals(actual.libDir(), expected.libDir(), "libDir"); + assertEquals(actual.modulesDir(), expected.modulesDir(), "modulesDir"); + assertEquals(actual.logsDir(), expected.logsDir(), "logsDir"); + assertEquals(actual.tmpDir(), expected.tmpDir(), "tmpDir"); } private static void assertEquals(Object actual, Object expected, String name) { diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index f3a3fc9f771d4..376f5016680e0 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -217,10 +217,10 @@ public NodeLock( final CheckedFunction pathFunction, final Function subPathMapping ) throws IOException { - dataPaths = new DataPath[environment.dataFiles().length]; + dataPaths = new DataPath[environment.dataDirs().length]; locks = new Lock[dataPaths.length]; try { - final Path[] dataPaths = environment.dataFiles(); + final Path[] dataPaths = environment.dataDirs(); for (int dirIndex = 0; dirIndex < dataPaths.length; dirIndex++) { Path dataDir = dataPaths[dirIndex]; Path dir = subPathMapping.apply(dataDir); @@ -269,9 +269,9 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce boolean success = false; try { - sharedDataPath = environment.sharedDataFile(); + sharedDataPath = environment.sharedDataDir(); - for (Path path : environment.dataFiles()) { + for (Path path : environment.dataDirs()) { if (Files.exists(path)) { // Call to toRealPath required to resolve symlinks. // We let it fall through to create directories to ensure the symlink @@ -289,7 +289,7 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce Locale.ROOT, "failed to obtain node locks, tried %s;" + " maybe these locations are not writable or multiple nodes were started on the same data path?", - Arrays.toString(environment.dataFiles()) + Arrays.toString(environment.dataDirs()) ); throw new IllegalStateException(message, e); } @@ -312,7 +312,7 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce } // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades - for (Path dataPath : environment.dataFiles()) { + for (Path dataPath : environment.dataDirs()) { final Path legacyNodesPath = dataPath.resolve("nodes"); if (Files.isRegularFile(legacyNodesPath) == false) { final String content = "written by Elasticsearch " @@ -351,7 +351,7 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings boolean upgradeNeeded = false; // check if we can do an auto-upgrade - for (Path path : environment.dataFiles()) { + for (Path path : environment.dataDirs()) { final Path nodesFolderPath = path.resolve("nodes"); if (Files.isDirectory(nodesFolderPath)) { final List nodeLockIds = new ArrayList<>(); @@ -394,7 +394,7 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings return false; } - logger.info("upgrading legacy data folders: {}", Arrays.toString(environment.dataFiles())); + logger.info("upgrading legacy data folders: {}", Arrays.toString(environment.dataDirs())); // acquire locks on legacy path for duration of upgrade (to ensure there is no older ES version running on this path) final NodeLock legacyNodeLock; @@ -405,7 +405,7 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings Locale.ROOT, "failed to obtain legacy node locks, tried %s;" + " maybe these locations are not writable or multiple nodes were started on the same data path?", - Arrays.toString(environment.dataFiles()) + Arrays.toString(environment.dataDirs()) ); throw new IllegalStateException(message, e); } @@ -496,7 +496,7 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings } // upgrade successfully completed, remove legacy nodes folders - IOUtils.rm(Stream.of(environment.dataFiles()).map(path -> path.resolve("nodes")).toArray(Path[]::new)); + IOUtils.rm(Stream.of(environment.dataDirs()).map(path -> path.resolve("nodes")).toArray(Path[]::new)); return true; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java index f940e87e51391..c1abf1f670756 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java @@ -23,8 +23,10 @@ public Set getFeatures() { public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field"); + private static final NodeFeature SYNONYMS_SET_LENIENT_ON_NON_EXISTING = new NodeFeature("index.synonyms_set_lenient_on_non_existing"); + @Override public Set getTestFeatures() { - return Set.of(LOGSDB_NO_HOST_NAME_FIELD); + return Set.of(LOGSDB_NO_HOST_NAME_FIELD, SYNONYMS_SET_LENIENT_ON_NON_EXISTING); } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 5512dffdda53e..baba9e94db7a7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -232,7 +232,8 @@ public IndexService( mapperMetrics ); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService); - if (indexSettings.getIndexSortConfig().hasIndexSort()) { + boolean sourceOnly = Boolean.parseBoolean(indexSettings.getSettings().get("index.source_only")); + if (indexSettings.getIndexSortConfig().hasIndexSort() && sourceOnly == false) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index d829ee41ef024..f4e6918043bd2 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -71,12 +71,14 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion V_7_15_0 = def(7_15_00_99, Version.LUCENE_8_9_0); public static final IndexVersion V_7_16_0 = def(7_16_00_99, Version.LUCENE_8_10_1); public static final IndexVersion V_7_17_0 = def(7_17_00_99, Version.LUCENE_8_11_1); + public static final IndexVersion V_7_17_19 = def(7_17_19_99, Version.LUCENE_8_11_3); public static final IndexVersion V_8_0_0 = def(8_00_00_99, Version.LUCENE_9_0_0); public static final IndexVersion V_8_1_0 = def(8_01_00_99, Version.LUCENE_9_0_0); public static final IndexVersion V_8_2_0 = def(8_02_00_99, Version.LUCENE_9_1_0); public static final IndexVersion V_8_3_0 = def(8_03_00_99, Version.LUCENE_9_2_0); public static final IndexVersion V_8_4_0 = def(8_04_00_99, Version.LUCENE_9_3_0); public static final IndexVersion V_8_5_0 = def(8_05_00_99, Version.LUCENE_9_4_1); + public static final IndexVersion V_8_5_3 = def(8_05_03_99, Version.LUCENE_9_4_2); public static final IndexVersion V_8_6_0 = def(8_06_00_99, Version.LUCENE_9_4_2); public static final IndexVersion V_8_7_0 = def(8_07_00_99, Version.LUCENE_9_5_0); public static final IndexVersion V_8_8_0 = def(8_08_00_99, Version.LUCENE_9_6_0); @@ -99,33 +101,34 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_LUCENE_9_9_1 = def(8_500_008, Version.LUCENE_9_9_1); public static final IndexVersion ES_VERSION_8_12_1 = def(8_500_009, Version.LUCENE_9_9_1); public static final IndexVersion UPGRADE_8_12_1_LUCENE_9_9_2 = def(8_500_010, Version.LUCENE_9_9_2); - public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); - public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); - public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); - public static final IndexVersion UPGRADE_TO_LUCENE_9_10 = def(8_503_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion TIME_SERIES_ROUTING_HASH_IN_ID = def(8_504_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_INT8_HNSW = def(8_505_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion DOC_VALUES_FOR_IGNORED_META_FIELD = def(8_505_00_1, Version.LUCENE_9_10_0); - public static final IndexVersion SOURCE_MAPPER_LOSSY_PARAMS_CHECK = def(8_506_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion SEMANTIC_TEXT_FIELD_TYPE = def(8_507_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion UPGRADE_TO_LUCENE_9_11 = def(8_508_00_0, Version.LUCENE_9_11_0); - public static final IndexVersion UNIQUE_TOKEN_FILTER_POS_FIX = def(8_509_00_0, Version.LUCENE_9_11_0); - public static final IndexVersion ADD_SECURITY_MIGRATION = def(8_510_00_0, Version.LUCENE_9_11_0); - public static final IndexVersion UPGRADE_TO_LUCENE_9_11_1 = def(8_511_00_0, Version.LUCENE_9_11_1); - public static final IndexVersion INDEX_SORTING_ON_NESTED = def(8_512_00_0, Version.LUCENE_9_11_1); - public static final IndexVersion LENIENT_UPDATEABLE_SYNONYMS = def(8_513_00_0, Version.LUCENE_9_11_1); - public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1); - public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1); - public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT = def(8_519_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID_BACKPORT = def(8_520_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0); - public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, Version.LUCENE_9_12_1); - public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_00_0, Version.LUCENE_9_12_1); - public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_00_0, Version.LUCENE_9_12_1); + public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_0_00, Version.LUCENE_9_9_1); + public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_0_00, Version.LUCENE_9_9_2); + public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_0_01, Version.LUCENE_9_9_2); + public static final IndexVersion UPGRADE_TO_LUCENE_9_10 = def(8_503_0_00, Version.LUCENE_9_10_0); + public static final IndexVersion TIME_SERIES_ROUTING_HASH_IN_ID = def(8_504_0_00, Version.LUCENE_9_10_0); + public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_INT8_HNSW = def(8_505_0_00, Version.LUCENE_9_10_0); + public static final IndexVersion DOC_VALUES_FOR_IGNORED_META_FIELD = def(8_505_0_01, Version.LUCENE_9_10_0); + public static final IndexVersion SOURCE_MAPPER_LOSSY_PARAMS_CHECK = def(8_506_0_00, Version.LUCENE_9_10_0); + public static final IndexVersion SEMANTIC_TEXT_FIELD_TYPE = def(8_507_0_00, Version.LUCENE_9_10_0); + public static final IndexVersion UPGRADE_TO_LUCENE_9_11 = def(8_508_0_00, Version.LUCENE_9_11_0); + public static final IndexVersion UNIQUE_TOKEN_FILTER_POS_FIX = def(8_509_0_00, Version.LUCENE_9_11_0); + public static final IndexVersion ADD_SECURITY_MIGRATION = def(8_510_0_00, Version.LUCENE_9_11_0); + public static final IndexVersion UPGRADE_TO_LUCENE_9_11_1 = def(8_511_0_00, Version.LUCENE_9_11_1); + public static final IndexVersion INDEX_SORTING_ON_NESTED = def(8_512_0_00, Version.LUCENE_9_11_1); + public static final IndexVersion LENIENT_UPDATEABLE_SYNONYMS = def(8_513_0_00, Version.LUCENE_9_11_1); + public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_0_00, Version.LUCENE_9_11_1); + public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_0_00, Version.LUCENE_9_11_1); + public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT = def(8_519_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID_BACKPORT = def(8_520_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_0_00, Version.LUCENE_9_12_0); + public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_0_00, Version.LUCENE_9_12_1); + public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_0_00, Version.LUCENE_9_12_1); + public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, Version.LUCENE_9_12_1); + /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ @@ -141,17 +144,17 @@ private static IndexVersion def(int id, Version luceneVersion) { * To add a new index version, add a new constant at the bottom of the list, above this comment. Don't add other lines, * comments, etc. The version id has the following layout: * - * M_NNN_SS_P + * M_NNN_S_PP * * M - The major version of Elasticsearch * NNN - The server version part - * SS - The serverless version part. It should always be 00 here, it is used by serverless only. - * P - The patch version part + * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. + * PP - The patch version part * * To determine the id of the next IndexVersion constant, do the following: * - Use the same major version, unless bumping majors * - Bump the server version part by 1, unless creating a patch version - * - Leave the serverless part as 00 + * - Leave the subsidiary part as 0 * - Bump the patch part if creating a patch version * * If a patch version is created, it should be placed sorted among the other existing constants. diff --git a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java index 462490a7fceb7..a518624fb0ab0 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -48,6 +48,7 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.apache.lucene.analysis.util.CSVUtil; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; @@ -66,7 +67,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.security.AccessControlException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -247,7 +247,7 @@ public static List getWordList( } } - final Path path = env.configFile().resolve(wordListPath); + final Path path = env.configDir().resolve(wordListPath); try { return loadWordList(path, removeComments); @@ -261,7 +261,7 @@ public static List getWordList( } catch (IOException ioe) { String message = Strings.format("IOException while reading %s: %s", settingPath, path); throw new IllegalArgumentException(message, ioe); - } catch (AccessControlException ace) { + } catch (SecurityException ace) { throw new IllegalArgumentException(Strings.format("Access denied trying to read file %s: %s", settingPath, path), ace); } } @@ -351,7 +351,7 @@ public static Reader getReaderFromFile(Environment env, String filePath, String if (filePath == null) { return null; } - final Path path = env.configFile().resolve(filePath); + final Path path = env.configDir().resolve(filePath); try { return Files.newBufferedReader(path, StandardCharsets.UTF_8); } catch (CharacterCodingException ex) { @@ -368,10 +368,39 @@ public static Reader getReaderFromFile(Environment env, String filePath, String } } - public static Reader getReaderFromIndex(String synonymsSet, SynonymsManagementAPIService synonymsManagementAPIService) { + public static Reader getReaderFromIndex( + String synonymsSet, + SynonymsManagementAPIService synonymsManagementAPIService, + boolean ignoreMissing + ) { final PlainActionFuture> synonymsLoadingFuture = new PlainActionFuture<>(); synonymsManagementAPIService.getSynonymSetRules(synonymsSet, synonymsLoadingFuture); - PagedResult results = synonymsLoadingFuture.actionGet(); + + PagedResult results; + + try { + results = synonymsLoadingFuture.actionGet(); + } catch (Exception e) { + if (ignoreMissing == false) { + throw e; + } + + boolean notFound = e instanceof ResourceNotFoundException; + String message = String.format( + Locale.ROOT, + "Synonyms set %s %s. Synonyms will not be applied to search results on indices that use this synonym set", + synonymsSet, + notFound ? "not found" : "could not be loaded" + ); + + if (notFound) { + logger.warn(message); + } else { + logger.error(message, e); + } + + results = new PagedResult<>(0, new SynonymRule[0]); + } SynonymRule[] synonymRules = results.pageResults(); StringBuilder sb = new StringBuilder(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java index 20154c20b3634..73a92869e31ba 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.engine; +import com.carrotsearch.hppc.IntArrayList; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; @@ -191,8 +193,28 @@ private Translog.Operation[] loadDocuments(List documentRecords) t maxDoc = leafReaderContext.reader().maxDoc(); } while (docRecord.docID() >= docBase + maxDoc); - leafFieldLoader = storedFieldLoader.getLoader(leafReaderContext, null); - leafSourceLoader = sourceLoader.leaf(leafReaderContext.reader(), null); + // TODO: instead of building an array, consider just checking whether doc ids are dense. + // Note, field loaders then would lose the ability to optionally eagerly loading values. + IntArrayList nextDocIds = new IntArrayList(); + for (int j = i; j < documentRecords.size(); j++) { + var record = documentRecords.get(j); + if (record.isTombstone()) { + continue; + } + int docID = record.docID(); + if (docID >= docBase + maxDoc) { + break; + } + int segmentDocID = docID - docBase; + nextDocIds.add(segmentDocID); + } + + // This computed doc ids arrays us used by stored field loader as a heuristic to determine whether to use a sequential + // stored field reader (which bulk loads stored fields and avoids decompressing the same blocks multiple times). For + // source loader, it is also used as a heuristic for bulk reading doc values (E.g. SingletonDocValuesLoader). + int[] nextDocIdArray = nextDocIds.toArray(); + leafFieldLoader = storedFieldLoader.getLoader(leafReaderContext, nextDocIdArray); + leafSourceLoader = sourceLoader.leaf(leafReaderContext.reader(), nextDocIdArray); setNextSourceMetadataReader(leafReaderContext); } int segmentDocID = docRecord.docID() - docBase; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 7da1e7d8a6790..1a12e4c7733a9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -15,6 +15,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; @@ -67,7 +68,7 @@ protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOEx return indexFieldData.load(context).getBytesValues(); } - protected void setScorer(Scorable scorer) {} + protected void setScorer(LeafReaderContext context, Scorable scorer) {} @Override public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { @@ -120,10 +121,43 @@ protected BinaryDocValues getBinaryDocValues(LeafReaderContext context, String f } @Override - public void setScorer(Scorable scorer) { - BytesRefFieldComparatorSource.this.setScorer(scorer); - } + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + LeafFieldComparator leafComparator = super.getLeafComparator(context); + // TopFieldCollector interacts with inter-segment concurrency by creating a FieldValueHitQueue per slice, each one with a + // specific instance of the FieldComparator. This ensures sequential execution across LeafFieldComparators returned by + // the same parent FieldComparator. That allows for effectively sharing the same instance of leaf comparator, like in this + // case in the Lucene code. That's fine dealing with sorting by field, but not when using script sorting, because we then + // need to set to Scorer to the specific leaf comparator, to make the _score variable available in sort scripts. The + // setScorer call happens concurrently across slices and needs to target the specific leaf context that is being searched. + return new LeafFieldComparator() { + @Override + public void setBottom(int slot) throws IOException { + leafComparator.setBottom(slot); + } + + @Override + public int compareBottom(int doc) throws IOException { + return leafComparator.compareBottom(doc); + } + + @Override + public int compareTop(int doc) throws IOException { + return leafComparator.compareTop(doc); + } + + @Override + public void copy(int slot, int doc) throws IOException { + leafComparator.copy(slot, doc); + } + @Override + public void setScorer(Scorable scorer) { + // this ensures that the scorer is set for the specific leaf comparator + // corresponding to the leaf context we are scoring + BytesRefFieldComparatorSource.this.setScorer(context, scorer); + } + }; + } }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index ae9ec46cf152a..c5fcb0207ce4d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -71,7 +71,7 @@ private NumericDoubleValues getNumericDocValues(LeafReaderContext context, doubl } } - protected void setScorer(Scorable scorer) {} + protected void setScorer(LeafReaderContext context, Scorable scorer) {} @Override public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { @@ -91,7 +91,7 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String @Override public void setScorer(Scorable scorer) { - DoubleValuesComparatorSource.this.setScorer(scorer); + DoubleValuesComparatorSource.this.setScorer(context, scorer); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index c38b5beeb55a0..6e00cc765bd8b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -180,12 +180,6 @@ protected Object parseSourceValue(Object value) { }; } - @Override - public BlockLoader blockLoader(BlockLoaderContext blContext) { - // Currently we can only load from source in ESQL - return blockLoaderFromSource(blContext); - } - protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 318e877c7ebb9..22b198b10a7ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -10,16 +10,12 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.Orientation; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.WellKnownBinary; -import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.lucene.spatial.Extent; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.io.IOException; -import java.nio.ByteOrder; import java.util.Map; import java.util.function.Function; @@ -75,29 +71,27 @@ public Orientation orientation() { @Override protected Object nullValueAsSource(T nullValue) { - // we don't support null value fors shapes + // we don't support null value for shapes return nullValue; } - @Override - public BlockLoader blockLoader(BlockLoaderContext blContext) { - return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS && isBoundsExtractionSupported() - ? new BoundsBlockLoader(name(), coordinateEncoder()) - : blockLoaderFromSource(blContext); - } - - protected abstract boolean isBoundsExtractionSupported(); - - protected abstract CoordinateEncoder coordinateEncoder(); - - // Visible for testing - static class BoundsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + protected static class BoundsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { private final String fieldName; - private final CoordinateEncoder encoder; - BoundsBlockLoader(String fieldName, CoordinateEncoder encoder) { + protected BoundsBlockLoader(String fieldName) { this.fieldName = fieldName; - this.encoder = encoder; + } + + protected void writeExtent(BlockLoader.IntBuilder builder, Extent extent) { + // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class + builder.beginPositionEntry(); + builder.appendInt(extent.top); + builder.appendInt(extent.bottom); + builder.appendInt(extent.negLeft); + builder.appendInt(extent.negRight); + builder.appendInt(extent.posLeft); + builder.appendInt(extent.posRight); + builder.endPositionEntry(); } @Override @@ -107,7 +101,7 @@ public BlockLoader.AllReader reader(LeafReaderContext context) throws IOExceptio public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { var binaryDocValues = context.reader().getBinaryDocValues(fieldName); var reader = new GeometryDocValueReader(); - try (var builder = factory.bytesRefs(docs.count())) { + try (var builder = factory.ints(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(binaryDocValues, docs.get(i), reader, builder); } @@ -119,27 +113,17 @@ public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { var binaryDocValues = context.reader().getBinaryDocValues(fieldName); var reader = new GeometryDocValueReader(); - read(binaryDocValues, docId, reader, (BytesRefBuilder) builder); + read(binaryDocValues, docId, reader, (IntBuilder) builder); } - private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueReader reader, BytesRefBuilder builder) + private void read(BinaryDocValues binaryDocValues, int doc, GeometryDocValueReader reader, IntBuilder builder) throws IOException { if (binaryDocValues.advanceExact(doc) == false) { builder.appendNull(); return; } reader.reset(binaryDocValues.binaryValue()); - var extent = reader.getExtent(); - // This is rather silly: an extent is already encoded as ints, but we convert it to Rectangle to - // preserve its properties as a WKB shape, only to convert it back to ints when we compute the - // aggregation. An obvious optimization would be to avoid this back-and-forth conversion. - var rectangle = new Rectangle( - encoder.decodeX(extent.minX()), - encoder.decodeX(extent.maxX()), - encoder.decodeY(extent.maxY()), - encoder.decodeY(extent.minY()) - ); - builder.appendBytesRef(new BytesRef(WellKnownBinary.toWKB(rectangle, ByteOrder.LITTLE_ENDIAN))); + writeExtent(builder, reader.getExtent()); } @Override @@ -151,7 +135,7 @@ public boolean canReuse(int startingDocID) { @Override public BlockLoader.Builder builder(BlockLoader.BlockFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); + return factory.ints(expectedCount); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 823170fcfc1b2..5ddaff8b961bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -74,6 +75,7 @@ import java.util.function.LongSupplier; import static org.elasticsearch.common.time.DateUtils.toLong; +import static org.elasticsearch.common.time.DateUtils.toLongMillis; /** A {@link FieldMapper} for dates. */ public final class DateFieldMapper extends FieldMapper { @@ -93,12 +95,13 @@ public final class DateFieldMapper extends FieldMapper { private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis") .withLocale(DEFAULT_LOCALE) .toDateMathParser(); + public static final NodeFeature INVALID_DATE_FIX = new NodeFeature("mapper.range.invalid_date_fix"); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE, DateMillisDocValuesField::new) { @Override public long convert(Instant instant) { - return instant.toEpochMilli(); + return toLongMillis(instant); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index af876c4b37e47..cb9601c22de8f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -62,6 +62,7 @@ public Set getFeatures() { public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed"); public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); + static final NodeFeature NPE_ON_DIMS_UPDATE_FIX = new NodeFeature("mapper.npe_on_dims_update_fix"); @Override public Set getTestFeatures() { @@ -79,7 +80,9 @@ public Set getTestFeatures() { SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, - ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX + ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX, + DateFieldMapper.INVALID_DATE_FIX, + NPE_ON_DIMS_UPDATE_FIX ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index b6539156c055d..77e70e4e6b646 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -800,7 +800,8 @@ public static boolean isMetadataFieldStatic(String fieldName) { * this method considers all mapper plugins */ public boolean isMetadataField(String field) { - return mapperRegistry.getMetadataMapperParsers(indexVersionCreated).containsKey(field); + var mapper = mappingLookup().getMapper(field); + return mapper instanceof MetadataFieldMapper; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 0a9028c480c1d..9656cb25e9f6c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -1130,7 +1130,7 @@ public boolean setIgnoredValues(Map( - "mode", - true, - () -> null, - (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), - m -> toType(m).enabled.explicit() ? null : toType(m).mode, + this.mode = new Parameter<>("mode", true, () -> null, (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), m -> { + var sfm = toType(m); + if (sfm.enabled.explicit()) { + return null; + } else if (sfm.serializeMode) { + return sfm.mode; + } else { + return null; + } + }, (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), - v -> v.toString().toLowerCase(Locale.ROOT) + v -> v != null ? v.toString().toLowerCase(Locale.ROOT) : null ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) // don't emit if `enabled` is configured .setSerializerCheck((includeDefaults, isConfigured, value) -> serializeMode && value != null); @@ -298,11 +304,20 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { return DEFAULT; } + SourceFieldMapper sourceFieldMapper; if (c.indexVersionCreated().onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)) { - return resolveStaticInstance(settingSourceMode); + sourceFieldMapper = resolveStaticInstance(settingSourceMode); } else { - return new SourceFieldMapper(settingSourceMode, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, true); + sourceFieldMapper = new SourceFieldMapper( + settingSourceMode, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + true + ); } + indexMode.validateSourceFieldMapper(sourceFieldMapper); + return sourceFieldMapper; }, c -> new Builder( c.getIndexSettings().getMode(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java index c8a8dd4b4a898..c114a1619c7a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java @@ -54,7 +54,7 @@ public final void write(XContentBuilder b) throws IOException { case 1: b.field(simpleName); write(b, values.get(0)); - return; + break; default: b.startArray(simpleName); for (Object value : values) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 6d01ea21a6478..251998c84b8b7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -147,7 +147,7 @@ public static class Builder extends FieldMapper.Builder { } return XContentMapValues.nodeIntegerValue(o); - }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null) + }, m -> toType(m).fieldType().dims, XContentBuilder::field, Objects::toString).setSerializerCheck((id, ic, v) -> v != null) .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current)) .addValidator(dims -> { if (dims == null) { @@ -1228,7 +1228,7 @@ public final int hashCode() { } } - private enum VectorIndexType { + public enum VectorIndexType { HNSW("hnsw", false) { @Override public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { @@ -2405,6 +2405,12 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf magnitudeReader = leafReader.getNumericDocValues(fullPath() + COSINE_MAGNITUDE_FIELD_SUFFIX); } return docId -> { + if (values.docID() > docId) { + return hasValue = false; + } + if (values.docID() == docId) { + return hasValue = true; + } hasValue = docId == values.advance(docId); hasMagnitude = hasValue && magnitudeReader != null && magnitudeReader.advanceExact(docId); return hasValue; @@ -2413,6 +2419,12 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf byteVectorValues = leafReader.getByteVectorValues(fullPath()); if (byteVectorValues != null) { return docId -> { + if (byteVectorValues.docID() > docId) { + return hasValue = false; + } + if (byteVectorValues.docID() == docId) { + return hasValue = true; + } hasValue = docId == byteVectorValues.advance(docId); return hasValue; }; @@ -2474,6 +2486,12 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf return null; } return docId -> { + if (values.docID() > docId) { + return hasValue = false; + } + if (values.docID() == docId) { + return hasValue = true; + } hasValue = docId == values.advance(docId); return hasValue; }; diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index ebd4bdffb0436..a18c49bb99321 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -156,6 +156,8 @@ import java.io.Closeable; import java.io.IOException; import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.nio.channels.ClosedByInterruptException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -413,7 +415,6 @@ public IndexShard( this.refreshFieldHasValueListener = new RefreshFieldHasValueListener(); this.relativeTimeInNanosSupplier = relativeTimeInNanosSupplier; this.indexCommitListener = indexCommitListener; - this.fieldInfos = FieldInfos.EMPTY; } public ThreadPool getThreadPool() { @@ -1011,12 +1012,26 @@ private Engine.IndexResult applyIndexOperation( return index(engine, operation); } - public void setFieldInfos(FieldInfos fieldInfos) { - this.fieldInfos = fieldInfos; + private static final VarHandle FIELD_INFOS; + + static { + try { + FIELD_INFOS = MethodHandles.lookup().findVarHandle(IndexShard.class, "fieldInfos", FieldInfos.class); + } catch (Exception e) { + throw new ExceptionInInitializerError(e); + } } public FieldInfos getFieldInfos() { - return fieldInfos; + var res = fieldInfos; + if (res == null) { + // don't replace field infos loaded via the refresh listener to avoid overwriting the field with an older version of the + // field infos when racing with a refresh + var read = loadFieldInfos(); + var existing = (FieldInfos) FIELD_INFOS.compareAndExchange(this, null, read); + return existing == null ? read : existing; + } + return res; } public static Engine.Index prepareIndex( @@ -4067,16 +4082,21 @@ public void beforeRefresh() {} @Override public void afterRefresh(boolean didRefresh) { - if (enableFieldHasValue && (didRefresh || fieldInfos == FieldInfos.EMPTY)) { - try (Engine.Searcher hasValueSearcher = getEngine().acquireSearcher("field_has_value")) { - setFieldInfos(FieldInfos.getMergedFieldInfos(hasValueSearcher.getIndexReader())); - } catch (AlreadyClosedException ignore) { - // engine is closed - no updated FieldInfos is fine - } + if (enableFieldHasValue && (didRefresh || fieldInfos == null)) { + FIELD_INFOS.setRelease(IndexShard.this, loadFieldInfos()); } } } + private FieldInfos loadFieldInfos() { + try (Engine.Searcher hasValueSearcher = getEngine().acquireSearcher("field_has_value")) { + return FieldInfos.getMergedFieldInfos(hasValueSearcher.getIndexReader()); + } catch (AlreadyClosedException ignore) { + // engine is closed - no update to FieldInfos is fine + } + return FieldInfos.EMPTY; + } + /** * Returns the shard-level field stats, which includes the number of segments in the latest NRT reader of this shard * and the total number of fields across those segments. diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 3dc5953e3d3d8..e76060b6c0dc8 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -22,6 +22,7 @@ import org.apache.lucene.store.SimpleFSLockFactory; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; @@ -36,6 +37,8 @@ public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory { + private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random"); + public static final Setting INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> { return switch (s) { case "native" -> NativeFSLockFactory.INSTANCE; @@ -67,12 +70,20 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index // Use Lucene defaults final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); if (primaryDirectory instanceof MMapDirectory mMapDirectory) { - return new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions)); + Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions)); + if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { + dir = disableRandomAdvice(dir); + } + return dir; } else { return primaryDirectory; } case MMAPFS: - return setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions); + Directory dir = setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions); + if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { + dir = disableRandomAdvice(dir); + } + return dir; case SIMPLEFS: case NIOFS: return new NIOFSDirectory(location, lockFactory); @@ -94,6 +105,23 @@ public static MMapDirectory setPreload(MMapDirectory mMapDirectory, LockFactory return mMapDirectory; } + /** + * Return a {@link FilterDirectory} around the provided {@link Directory} that forcefully disables {@link IOContext#RANDOM random + * access}. + */ + static Directory disableRandomAdvice(Directory dir) { + return new FilterDirectory(dir) { + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + if (context.randomAccess) { + context = IOContext.READ; + } + assert context.randomAccess == false; + return super.openInput(name, context); + } + }; + } + /** * Returns true iff the directory is a hybrid fs directory */ diff --git a/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java index 29161814e7724..6ae35df7a02e9 100644 --- a/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/AssociatedIndexDescriptor.java @@ -13,6 +13,7 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.indices.system.IndexPatternMatcher; import java.util.List; import java.util.Objects; diff --git a/server/src/main/java/org/elasticsearch/indices/IndexPatternMatcher.java b/server/src/main/java/org/elasticsearch/indices/IndexMatcher.java similarity index 70% rename from server/src/main/java/org/elasticsearch/indices/IndexPatternMatcher.java rename to server/src/main/java/org/elasticsearch/indices/IndexMatcher.java index 93bc1430de705..48e0985caa863 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndexPatternMatcher.java +++ b/server/src/main/java/org/elasticsearch/indices/IndexMatcher.java @@ -14,23 +14,15 @@ import java.util.List; /** - * An IndexPatternMatcher holds an index pattern in a string and, given a - * {@link Metadata} object, can return a list of index names matching that pattern. + * An IndexMatcher given a {@link Metadata} object, can return a list of index names matching that pattern. */ -public interface IndexPatternMatcher { - /** - * @return A pattern, either with a wildcard or simple regex, describing indices that are - * related to a system feature. Such indices may be system indices or associated - * indices. - */ - String getIndexPattern(); - +public interface IndexMatcher { /** * Retrieves a list of all indices which match this descriptor's pattern. Implementations * may include other special information when matching indices, such as aliases. - * + *

* This cannot be done via {@link org.elasticsearch.cluster.metadata.IndexNameExpressionResolver} because that class can only handle - * simple wildcard expressions, but system index name patterns may use full Lucene regular expression syntax, + * simple wildcard expressions, but system index name patterns may use full Lucene regular expression syntax. * * @param metadata The current metadata to get the list of matching indices from * @return A list of index names that match this descriptor diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 3f44ad06a0cd5..adbe061717474 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -251,6 +251,7 @@ public class IndicesService extends AbstractLifecycleComponent private final Map recoveryStateFactories; private final IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListeners; final AbstractRefCounted indicesRefCount; // pkg-private for testing + private final CountDownLatch stopLatch = new CountDownLatch(1); private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; private volatile boolean allowExpensiveQueries; @@ -397,6 +398,7 @@ public ClusterService clusterService() { @Override protected void doStop() { + stopLatch.countDown(); clusterService.removeApplier(timestampFieldMapperService); timestampFieldMapperService.doStop(); @@ -1414,7 +1416,15 @@ public void processPendingDeletes(Index index, IndexSettings indexSettings, Time } if (remove.isEmpty() == false) { logger.warn("{} still pending deletes present for shards {} - retrying", index, remove.toString()); - Thread.sleep(sleepTime); + if (stopLatch.await(sleepTime, TimeUnit.MILLISECONDS)) { + logger.info( + "Indices service stopped. {} aborting pending deletes after [{}] for shards {}", + index, + TimeValue.timeValueNanos(System.nanoTime() - startTimeNS), + remove.toString() + ); + break; + } sleepTime = Math.min(maxSleepTimeMs, sleepTime * 2); // increase the sleep time gradually logger.debug("{} schedule pending delete retry after {} ms", index, sleepTime); } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java index 9a78556f9239b..db239c1817817 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java @@ -9,17 +9,18 @@ package org.elasticsearch.indices; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.index.Index; +import org.elasticsearch.indices.system.SystemResourceDescriptor; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; - -import static org.elasticsearch.indices.AssociatedIndexDescriptor.buildAutomaton; +import java.util.stream.Stream; /** * Describes a {@link DataStream} that is reserved for use by a system feature. @@ -44,7 +45,7 @@ *

The descriptor also provides names for the thread pools that Elasticsearch should use to read, search, or modify the descriptor’s * indices. */ -public class SystemDataStreamDescriptor { +public class SystemDataStreamDescriptor implements SystemResourceDescriptor { private final String dataStreamName; private final String description; @@ -52,8 +53,8 @@ public class SystemDataStreamDescriptor { private final ComposableIndexTemplate composableIndexTemplate; private final Map componentTemplates; private final List allowedElasticProductOrigins; + private final String origin; private final ExecutorNames executorNames; - private final CharacterRunAutomaton characterRunAutomaton; /** * Creates a new descriptor for a system data descriptor @@ -66,6 +67,7 @@ public class SystemDataStreamDescriptor { * {@link ComposableIndexTemplate} * @param allowedElasticProductOrigins a list of product origin values that are allowed to access this data stream if the * type is {@link Type#EXTERNAL}. Must not be {@code null} + * @param origin specifies the origin to use when creating or updating the data stream * @param executorNames thread pools that should be used for operations on the system data stream */ public SystemDataStreamDescriptor( @@ -75,6 +77,7 @@ public SystemDataStreamDescriptor( ComposableIndexTemplate composableIndexTemplate, Map componentTemplates, List allowedElasticProductOrigins, + String origin, ExecutorNames executorNames ) { this.dataStreamName = Objects.requireNonNull(dataStreamName, "dataStreamName must be specified"); @@ -96,8 +99,7 @@ public SystemDataStreamDescriptor( throw new IllegalArgumentException("External system data stream without allowed products is not a valid combination"); } this.executorNames = Objects.nonNull(executorNames) ? executorNames : ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS; - - this.characterRunAutomaton = new CharacterRunAutomaton(buildAutomaton(backingIndexPatternForDataStream(this.dataStreamName))); + this.origin = origin; } public String getDataStreamName() { @@ -110,7 +112,16 @@ public String getDataStreamName() { * @return List of names of backing indices */ public List getBackingIndexNames(Metadata metadata) { - return metadata.indices().keySet().stream().filter(this.characterRunAutomaton::run).toList(); + DataStream dataStream = metadata.dataStreams().get(dataStreamName); + if (dataStream == null) { + return Collections.emptyList(); + } + return Stream.concat(dataStream.getIndices().stream(), dataStream.getFailureIndices().stream()).map(Index::getName).toList(); + } + + @Override + public List getMatchingIndices(Metadata metadata) { + return getBackingIndexNames(metadata); } public String getDescription() { @@ -121,6 +132,17 @@ public ComposableIndexTemplate getComposableIndexTemplate() { return composableIndexTemplate; } + @Override + public String getOrigin() { + return origin; + } + + @Override + public boolean isAutomaticallyManaged() { + return true; + } + + @Override public boolean isExternal() { return type == Type.EXTERNAL; } @@ -130,9 +152,10 @@ public String getBackingIndexPattern() { } private static String backingIndexPatternForDataStream(String dataStream) { - return DataStream.BACKING_INDEX_PREFIX + dataStream + "-*"; + return ".(migrated-)?[fd]s-" + dataStream + "-*"; } + @Override public List getAllowedElasticProductOrigins() { return allowedElasticProductOrigins; } @@ -145,6 +168,7 @@ public Map getComponentTemplates() { * Get the names of the thread pools that should be used for operations on this data stream. * @return Names for get, search, and write executors. */ + @Override public ExecutorNames getThreadPoolNames() { return this.executorNames; } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 5fa8221d263b6..dfae99683abb9 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -14,7 +14,6 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -27,6 +26,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.indices.system.IndexPatternMatcher; +import org.elasticsearch.indices.system.SystemResourceDescriptor; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -101,7 +102,7 @@ * A system index that is fully internal to Elasticsearch will not allow any product origins; such an index is fully "locked down," * and in general can only be changed by restoring feature states from snapshots. */ -public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable { +public class SystemIndexDescriptor implements IndexPatternMatcher, SystemResourceDescriptor, Comparable { public static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(); @@ -152,7 +153,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable @@ -317,7 +318,7 @@ protected SystemIndexDescriptor( } Objects.requireNonNull(allowedElasticProductOrigins, "allowedProductOrigins must not be null"); - if (type.isInternal() && allowedElasticProductOrigins.isEmpty() == false) { + if (type.isExternal() == false && allowedElasticProductOrigins.isEmpty() == false) { throw new IllegalArgumentException("Allowed origins are not valid for internal system indices"); } else if (type.isExternal() && allowedElasticProductOrigins.isEmpty()) { throw new IllegalArgumentException("External system indices without allowed products is not a valid combination"); @@ -464,9 +465,7 @@ public List getMatchingIndices(Metadata metadata) { return metadata.indices().keySet().stream().filter(this::matchesIndexPattern).toList(); } - /** - * @return A short description of the purpose of this system index. - */ + @Override public String getDescription() { return description; } @@ -495,26 +494,17 @@ public int getIndexFormat() { return this.indexFormat; } - public String getMappingsNodeVersionMetaKey() { - assert isAutomaticallyManaged() : "Do not request version meta keys for unmanaged system indices"; - return this.mappingsNodeVersionMetaKey; - } - public Version getMinimumNodeVersion() { assert isAutomaticallyManaged() : "Do not request version minimum node version for unmanaged system indices"; return minimumNodeVersion; } + @Override public boolean isAutomaticallyManaged() { return type.isManaged(); } - /** - * Get an origin string suitable for use in an {@link org.elasticsearch.client.internal.OriginSettingClient}. See - * {@link Builder#setOrigin(String)} for more information. - * - * @return an origin string to use for sub-requests - */ + @Override public String getOrigin() { // TODO[wrb]: most unmanaged system indices do not set origins; could we assert on that here? return this.origin; @@ -525,20 +515,12 @@ public boolean hasDynamicMappings() { return this.hasDynamicMappings; } + @Override public boolean isExternal() { return type.isExternal(); } - public boolean isInternal() { - return type.isInternal(); - } - - /** - * Requests from these products, if made with the proper security credentials, are allowed non-deprecated access to this descriptor's - * indices. (Product names may be specified in requests with the - * {@link org.elasticsearch.tasks.Task#X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER}). - * @return A list of product names. - */ + @Override public List getAllowedElasticProductOrigins() { return allowedElasticProductOrigins; } @@ -662,6 +644,7 @@ public SystemIndexDescriptor getDescriptorCompatibleWith(MappingsVersion version * @return The names of thread pools that should be used for operations on this * system index. */ + @Override public ExecutorNames getThreadPoolNames() { return this.executorNames; } @@ -714,10 +697,6 @@ public boolean isExternal() { public boolean isManaged() { return managed; } - - public boolean isInternal() { - return external == false; - } } /** diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index d059f8ff3cc92..889af91327896 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -16,6 +16,7 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.MinimizationOperations; import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; @@ -25,6 +26,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.TriConsumer; @@ -37,6 +39,11 @@ import org.elasticsearch.core.Predicates; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.indices.system.IndexPatternMatcher; +import org.elasticsearch.indices.system.SystemResourceDescriptor; +import org.elasticsearch.node.Node; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.snapshots.SnapshotsService; @@ -70,7 +77,7 @@ * from the user index space for a few reasons. In some cases, the indices contain information that should be hidden from users. But, * more generally, we want to protect these indices and data streams from being inadvertently modified or deleted. * - *

The system resources are grouped by feature, using the {@link SystemIndices.Feature} class. Most features will be loaded from + *

The system resources are grouped by feature, using the {@link Feature} class. Most features will be loaded from * instances of {@link SystemIndexPlugin}; any other features will be described in this class. Features may be retrieved by name or * iterated over (see {@link #getFeature(String)} and {@link #getFeatures()}). Each Feature provides collections of * {@link SystemIndexDescriptor}s or {@link SystemDataStreamDescriptor}s. These descriptors define their resources by means of patterns. @@ -81,7 +88,7 @@ *

For more information about the expected behavior of system indices, see {@link SystemIndexDescriptor}. For more information about * the expected behavior of system data streams, see {@link SystemDataStreamDescriptor}. * - *

The SystemIndices object is constructed during {@link org.elasticsearch.node.Node} startup, and is not modified after construction. + *

The SystemIndices object is constructed during {@link Node} startup, and is not modified after construction. * In other words, the set of system resources will be consistent over the lifetime of a node. * *

System resources will specify thread pools for reads, writes, and searches. This can ensure that system-critical operations, such @@ -111,7 +118,16 @@ public class SystemIndices { public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed"; public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin"; - public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-9"; + + /** + * These versions should be set to current major and current major's index version + */ + public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0; + public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0; + public static final String MIGRATE_SYSTEM_INDEX_CAUSE = "migrate-system-index"; + + private static final int UPGRADED_TO_VERSION = NO_UPGRADE_REQUIRED_VERSION.major + 1; + public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION; private static final Automaton EMPTY = Automata.makeEmpty(); @@ -221,7 +237,7 @@ private static void checkForDuplicateAliases(Collection d final List duplicateAliases = aliasCounts.entrySet() .stream() .filter(entry -> entry.getValue() > 1) - .map(Map.Entry::getKey) + .map(Entry::getKey) .sorted() .toList(); @@ -309,7 +325,7 @@ public boolean isSystemDataStream(String name) { /** * Determines whether the provided name matches that of an index that backs a system data stream. Backing indices * for system data streams are marked as "system" in their metadata (see {@link - * org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService}) and receive the same protections as the + * SystemIndexMetadataUpgradeService}) and receive the same protections as the * system data stream. */ public boolean isSystemIndexBackingDataStream(String name) { @@ -347,6 +363,7 @@ public ExecutorSelector getExecutorSelector() { /** * Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name. + * Does not take into account system data streams and their backing indices. * @param name the name of the index * @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found */ @@ -355,7 +372,7 @@ public ExecutorSelector getExecutorSelector() { } @Nullable - static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) { + private static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) { SystemIndexDescriptor matchingDescriptor = null; for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) { if (systemIndexDescriptor.matchesIndexPattern(name)) { @@ -702,7 +719,7 @@ private static Map buildFeatureMap(List features) { return Map.copyOf(map); } - Collection getSystemIndexDescriptors() { + public Collection getSystemIndexDescriptors() { return this.featureDescriptors.values().stream().flatMap(f -> f.getIndexDescriptors().stream()).toList(); } @@ -869,6 +886,14 @@ public Collection getDataStreamDescriptors() { return dataStreamDescriptors; } + /** + * Returns descriptors of all system resources - indices and data streams. + * Doesn't include associated indices {@link AssociatedIndexDescriptor}. + */ + public Collection getSystemResourceDescriptors() { + return Stream.concat(indexDescriptors.stream(), dataStreamDescriptors.stream()).toList(); + } + public Collection getAssociatedIndexDescriptors() { return associatedIndexDescriptors; } diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java index 886c5e4bf6d3a..ab6e20c39f720 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java @@ -99,7 +99,7 @@ public HunspellService(final Settings settings, final Environment env, final Map try { return loadDictionary(locale, settings, env); } catch (Exception e) { - throw new IllegalStateException("failed to load hunspell dictionary for locale: " + locale, e); + throw new IllegalArgumentException("failed to load hunspell dictionary for locale: " + locale, e); } }; if (HUNSPELL_LAZY_LOAD.get(settings) == false) { @@ -122,7 +122,7 @@ public Dictionary getDictionary(String locale) { } private static Path resolveHunspellDirectory(Environment env) { - return env.configFile().resolve("hunspell"); + return env.configDir().resolve("hunspell"); } /** @@ -193,7 +193,7 @@ private Dictionary loadDictionary(String locale, Settings nodeSettings, Environm affixStream = Files.newInputStream(affixFiles[0]); - try (Directory tmp = new NIOFSDirectory(env.tmpFile())) { + try (Directory tmp = new NIOFSDirectory(env.tmpDir())) { return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase); } diff --git a/server/src/main/java/org/elasticsearch/indices/system/IndexPatternMatcher.java b/server/src/main/java/org/elasticsearch/indices/system/IndexPatternMatcher.java new file mode 100644 index 0000000000000..5526f437d80c8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/system/IndexPatternMatcher.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.indices.system; + +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.indices.IndexMatcher; + +/** + * An IndexPatternMatcher holds an index pattern in a string and, given a + * {@link Metadata} object, can return a list of index names matching that pattern. + */ +public interface IndexPatternMatcher extends IndexMatcher { + /** + * @return A pattern, either with a wildcard or simple regex, describing indices that are + * related to a system feature. Such indices may be system indices or associated + * indices. + */ + String getIndexPattern(); + +} diff --git a/server/src/main/java/org/elasticsearch/indices/system/SystemResourceDescriptor.java b/server/src/main/java/org/elasticsearch/indices/system/SystemResourceDescriptor.java new file mode 100644 index 0000000000000..023b54dec989d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/system/SystemResourceDescriptor.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.indices.system; + +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.IndexMatcher; +import org.elasticsearch.indices.SystemIndexDescriptor; + +import java.util.List; + +public interface SystemResourceDescriptor extends IndexMatcher { + /** + * @return A short description of the purpose of this system resource. + */ + String getDescription(); + + boolean isAutomaticallyManaged(); + + /** + * Get an origin string suitable for use in an {@link org.elasticsearch.client.internal.OriginSettingClient}. See + * {@link SystemIndexDescriptor.Builder#setOrigin(String)} for more information. + * + * @return an origin string to use for sub-requests + */ + String getOrigin(); + + boolean isExternal(); + + /** + * Requests from these products, if made with the proper security credentials, are allowed non-deprecated access to this descriptor's + * indices. (Product names may be specified in requests with the + * {@link org.elasticsearch.tasks.Task#X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER}). + * @return A list of product names. + */ + List getAllowedElasticProductOrigins(); + + /** + * @return The names of thread pools that should be used for operations on this system index. + */ + ExecutorNames getThreadPoolNames(); +} diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index e1ebd8bb81ff4..de1925cb641e9 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -241,4 +241,10 @@ default void defaultConfigs(ActionListener> defaultsListener) { default void updateModelsWithDynamicFields(List model, ActionListener> listener) { listener.onResponse(model); } + + /** + * Called after the Elasticsearch node has completed its start up. This allows the service to perform initialization + * after ensuring the node's internals are set up (for example if this ensures the internal ES client is ready for use). + */ + default void onNodeStarted() {} } diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java index f1ce94173a550..8ef9b59f5545a 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java @@ -41,6 +41,16 @@ public void init(Client client) { services.values().forEach(s -> s.init(client)); } + public void onNodeStarted() { + for (var service : services.values()) { + try { + service.onNodeStarted(); + } catch (Exception e) { + // ignore + } + } + } + public Map getServices() { return services; } diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java index 97a68d9807688..86f855a13c87e 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java @@ -541,18 +541,38 @@ public static TemplateScript.Factory compileTemplate( Script script = new Script(ScriptType.INLINE, DEFAULT_TEMPLATE_LANG, propertyValue, Map.of()); return scriptService.compile(script, TemplateScript.CONTEXT); } else { - return (params) -> new TemplateScript(params) { - @Override - public String execute() { - return propertyValue; - } - }; + return new ConstantTemplateScriptFactory(propertyValue); } } catch (Exception e) { throw ConfigurationUtils.newConfigurationException(processorType, processorTag, propertyName, e); } } + /** + * A 'template script' that ignores the model to which it is applied and just always returns a constant String. + *

+ * Having a separate named class for this allows for some hot code paths to pre-apply the 'template script' statically, + * rather than bothering to invoke it per-document. Note that this is probably only useful if something expensive + * is being done with the result of calling the script, and the code can then avoid doing that thing per-document. + */ + public static class ConstantTemplateScriptFactory implements TemplateScript.Factory { + final TemplateScript script; + + private ConstantTemplateScriptFactory(String value) { + this.script = new TemplateScript(Map.of()) { + @Override + public String execute() { + return value; + } + }; + } + + @Override + public TemplateScript newInstance(Map params) { + return script; + } + } + private static void addMetadataToException( ElasticsearchException exception, String processorType, diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 97cd738eaf5bd..fa850fe07bd6e 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -82,7 +82,7 @@ public final class IngestDocument { * of the pipeline was that the _index value did not change and so only 'foo' would appear * in the index history. */ - private Set indexHistory = new LinkedHashSet<>(); + private final Set indexHistory = new LinkedHashSet<>(); private boolean doNoSelfReferencesCheck = false; private boolean reroute = false; @@ -237,9 +237,7 @@ public byte[] getFieldValueAsBytes(String path, boolean ignoreMissing) { } else if (object instanceof String string) { return Base64.getDecoder().decode(string); } else { - throw new IllegalArgumentException( - "Content field [" + path + "] of unknown type [" + object.getClass().getName() + "], must be string or byte array" - ); + throw new IllegalArgumentException(Errors.notStringOrByteArray(path, object)); } } @@ -267,51 +265,42 @@ public boolean hasField(String path, boolean failOutOfRange) { String pathElement = fieldPath.pathElements[i]; if (context == null) { return false; - } - if (context instanceof Map map) { + } else if (context instanceof Map map) { context = map.get(pathElement); } else if (context instanceof List list) { + int index; try { - int index = Integer.parseInt(pathElement); - if (index < 0 || index >= list.size()) { - if (failOutOfRange) { - throw new IllegalArgumentException( - "[" - + index - + "] is out of bounds for array with length [" - + list.size() - + "] as part of path [" - + path - + "]" - ); - } else { - return false; - } - } - context = list.get(index); + index = Integer.parseInt(pathElement); } catch (NumberFormatException e) { return false; } - + if (index < 0 || index >= list.size()) { + if (failOutOfRange) { + throw new IllegalArgumentException(Errors.outOfBounds(path, index, list.size())); + } else { + return false; + } + } else { + context = list.get(index); + } } else { return false; } } String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; - if (context instanceof Map map) { + if (context == null) { + return false; + } else if (context instanceof Map map) { return map.containsKey(leafKey); - } - if (context instanceof List list) { + } else if (context instanceof List list) { try { int index = Integer.parseInt(leafKey); if (index >= 0 && index < list.size()) { return true; } else { if (failOutOfRange) { - throw new IllegalArgumentException( - "[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]" - ); + throw new IllegalArgumentException(Errors.outOfBounds(path, index, list.size())); } else { return false; } @@ -319,8 +308,9 @@ public boolean hasField(String path, boolean failOutOfRange) { } catch (NumberFormatException e) { return false; } + } else { + return false; } - return false; } /** @@ -341,79 +331,58 @@ public void removeField(String path) { } String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; - if (context instanceof Map map) { + if (context == null) { + throw new IllegalArgumentException(Errors.cannotRemove(path, leafKey, null)); + } else if (context instanceof Map map) { if (map.containsKey(leafKey)) { map.remove(leafKey); - return; + } else { + throw new IllegalArgumentException(Errors.notPresent(path, leafKey)); } - throw new IllegalArgumentException("field [" + leafKey + "] not present as part of path [" + path + "]"); - } - if (context instanceof List list) { + } else if (context instanceof List list) { int index; try { index = Integer.parseInt(leafKey); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "[" + leafKey + "] is not an integer, cannot be used as an index as part of path [" + path + "]", - e - ); + throw new IllegalArgumentException(Errors.notInteger(path, leafKey), e); } if (index < 0 || index >= list.size()) { - throw new IllegalArgumentException( - "[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]" - ); + throw new IllegalArgumentException(Errors.outOfBounds(path, index, list.size())); + } else { + list.remove(index); } - list.remove(index); - return; - } - - if (context == null) { - throw new IllegalArgumentException("cannot remove [" + leafKey + "] from null as part of path [" + path + "]"); + } else { + throw new IllegalArgumentException(Errors.cannotRemove(path, leafKey, context)); } - throw new IllegalArgumentException( - "cannot remove [" + leafKey + "] from object of type [" + context.getClass().getName() + "] as part of path [" + path + "]" - ); } private static ResolveResult resolve(String pathElement, String fullPath, Object context) { if (context == null) { - return ResolveResult.error("cannot resolve [" + pathElement + "] from null as part of path [" + fullPath + "]"); - } - if (context instanceof Map) { + return ResolveResult.error(Errors.cannotResolve(fullPath, pathElement, null)); + } else if (context instanceof Map) { @SuppressWarnings("unchecked") Map map = (Map) context; Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get if (object == NOT_FOUND) { - return ResolveResult.error("field [" + pathElement + "] not present as part of path [" + fullPath + "]"); + return ResolveResult.error(Errors.notPresent(fullPath, pathElement)); } else { return ResolveResult.success(object); } - } - if (context instanceof List list) { + } else if (context instanceof List list) { int index; try { index = Integer.parseInt(pathElement); } catch (NumberFormatException e) { - return ResolveResult.error( - "[" + pathElement + "] is not an integer, cannot be used as an index as part of path [" + fullPath + "]" - ); + return ResolveResult.error(Errors.notInteger(fullPath, pathElement)); } if (index < 0 || index >= list.size()) { - return ResolveResult.error( - "[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + fullPath + "]" - ); + return ResolveResult.error(Errors.outOfBounds(fullPath, index, list.size())); + } else { + return ResolveResult.success(list.get(index)); } - return ResolveResult.success(list.get(index)); + } else { + return ResolveResult.error(Errors.cannotResolve(fullPath, pathElement, context)); } - return ResolveResult.error( - "cannot resolve [" - + pathElement - + "] from object of type [" - + context.getClass().getName() - + "] as part of path [" - + fullPath - + "]" - ); } /** @@ -514,7 +483,6 @@ public void setFieldValue(String path, ValueSource valueSource, boolean ignoreEm return; } } - setFieldValue(path, value); } @@ -539,7 +507,6 @@ public void setFieldValue(String path, Object value, boolean ignoreEmptyValue) { } } } - setFieldValue(path, value); } @@ -549,9 +516,8 @@ private void setFieldValue(String path, Object value, boolean append, boolean al for (int i = 0; i < fieldPath.pathElements.length - 1; i++) { String pathElement = fieldPath.pathElements[i]; if (context == null) { - throw new IllegalArgumentException("cannot resolve [" + pathElement + "] from null as part of path [" + path + "]"); - } - if (context instanceof Map) { + throw new IllegalArgumentException(Errors.cannotResolve(path, pathElement, null)); + } else if (context instanceof Map) { @SuppressWarnings("unchecked") Map map = (Map) context; Object object = map.getOrDefault(pathElement, NOT_FOUND); // getOrDefault is faster than containsKey + get @@ -567,35 +533,22 @@ private void setFieldValue(String path, Object value, boolean append, boolean al try { index = Integer.parseInt(pathElement); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "[" + pathElement + "] is not an integer, cannot be used as an index as part of path [" + path + "]", - e - ); + throw new IllegalArgumentException(Errors.notInteger(path, pathElement), e); } if (index < 0 || index >= list.size()) { - throw new IllegalArgumentException( - "[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]" - ); + throw new IllegalArgumentException(Errors.outOfBounds(path, index, list.size())); + } else { + context = list.get(index); } - context = list.get(index); } else { - throw new IllegalArgumentException( - "cannot resolve [" - + pathElement - + "] from object of type [" - + context.getClass().getName() - + "] as part of path [" - + path - + "]" - ); + throw new IllegalArgumentException(Errors.cannotResolve(path, pathElement, context)); } } String leafKey = fieldPath.pathElements[fieldPath.pathElements.length - 1]; if (context == null) { - throw new IllegalArgumentException("cannot set [" + leafKey + "] with null parent as part of path [" + path + "]"); - } - if (context instanceof Map) { + throw new IllegalArgumentException(Errors.cannotSet(path, leafKey, null)); + } else if (context instanceof Map) { @SuppressWarnings("unchecked") Map map = (Map) context; if (append) { @@ -613,42 +566,30 @@ private void setFieldValue(String path, Object value, boolean append, boolean al return; } map.put(leafKey, value); - } else if (context instanceof List) { + } else if (context instanceof List) { @SuppressWarnings("unchecked") List list = (List) context; int index; try { index = Integer.parseInt(leafKey); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "[" + leafKey + "] is not an integer, cannot be used as an index as part of path [" + path + "]", - e - ); + throw new IllegalArgumentException(Errors.notInteger(path, leafKey), e); } if (index < 0 || index >= list.size()) { - throw new IllegalArgumentException( - "[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]" - ); - } - if (append) { - Object object = list.get(index); - Object newList = appendValues(object, value, allowDuplicates); - if (newList != object) { - list.set(index, newList); + throw new IllegalArgumentException(Errors.outOfBounds(path, index, list.size())); + } else { + if (append) { + Object object = list.get(index); + Object newList = appendValues(object, value, allowDuplicates); + if (newList != object) { + list.set(index, newList); + } + return; } - return; + list.set(index, value); } - list.set(index, value); } else { - throw new IllegalArgumentException( - "cannot set [" - + leafKey - + "] with parent object of type [" - + context.getClass().getName() - + "] as part of path [" - + path - + "]" - ); + throw new IllegalArgumentException(Errors.cannotSet(path, leafKey, context)); } } @@ -705,9 +646,7 @@ private static T cast(String path, Object object, Class clazz) { if (clazz.isInstance(object)) { return clazz.cast(object); } - throw new IllegalArgumentException( - "field [" + path + "] of type [" + object.getClass().getName() + "] cannot be cast to [" + clazz.getName() + "]" - ); + throw new IllegalArgumentException(Errors.cannotCast(path, object, clazz)); } /** @@ -823,15 +762,12 @@ public static Set getAllFields(Map input) { @SuppressWarnings("unchecked") private static Set getAllFields(Map input, String prefix) { Set allFields = Sets.newHashSet(); - input.forEach((k, v) -> { allFields.add(prefix + k); - if (v instanceof Map mapValue) { allFields.addAll(getAllFields((Map) mapValue, prefix + k + ".")); } }); - return allFields; } @@ -1052,24 +988,13 @@ public Object initialContext(IngestDocument document) { } } - private static class ResolveResult { - boolean wasSuccessful; - String errorMessage; - Object resolvedObject; - + private record ResolveResult(boolean wasSuccessful, Object resolvedObject, String errorMessage) { static ResolveResult success(Object resolvedObject) { - ResolveResult result = new ResolveResult(); - result.wasSuccessful = true; - result.resolvedObject = resolvedObject; - return result; + return new ResolveResult(true, resolvedObject, null); } static ResolveResult error(String errorMessage) { - ResolveResult result = new ResolveResult(); - result.wasSuccessful = false; - result.errorMessage = errorMessage; - return result; - + return new ResolveResult(false, null, errorMessage); } } @@ -1150,4 +1075,57 @@ public Set> entrySet() { throw new UnsupportedOperationException(); } } + + private static final class Errors { + private Errors() { + // utility class + } + + private static String cannotCast(String path, Object value, Class clazz) { + return "field [" + path + "] of type [" + value.getClass().getName() + "] cannot be cast to [" + clazz.getName() + "]"; + } + + private static String cannotRemove(String path, String key, Object value) { + if (value == null) { + return "cannot remove [" + key + "] from null as part of path [" + path + "]"; + } else { + final String type = value.getClass().getName(); + return "cannot remove [" + key + "] from object of type [" + type + "] as part of path [" + path + "]"; + } + } + + private static String cannotResolve(String path, String key, Object value) { + if (value == null) { + return "cannot resolve [" + key + "] from null as part of path [" + path + "]"; + } else { + final String type = value.getClass().getName(); + return "cannot resolve [" + key + "] from object of type [" + type + "] as part of path [" + path + "]"; + } + } + + private static String cannotSet(String path, String key, Object value) { + if (value == null) { + return "cannot set [" + key + "] with null parent as part of path [" + path + "]"; + } else { + final String type = value.getClass().getName(); + return "cannot set [" + key + "] with parent object of type [" + type + "] as part of path [" + path + "]"; + } + } + + private static String outOfBounds(String path, int index, int length) { + return "[" + index + "] is out of bounds for array with length [" + length + "] as part of path [" + path + "]"; + } + + private static String notInteger(String path, String key) { + return "[" + key + "] is not an integer, cannot be used as an index as part of path [" + path + "]"; + } + + private static String notPresent(String path, String key) { + return "field [" + key + "] not present as part of path [" + path + "]"; + } + + private static String notStringOrByteArray(String path, Object value) { + return "Content field [" + path + "] of unknown type [" + value.getClass().getName() + "], must be string or byte array"; + } + } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestPipelineMetric.java b/server/src/main/java/org/elasticsearch/ingest/IngestPipelineMetric.java index 4b9c23e3be567..a3cb3e4d90b64 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestPipelineMetric.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestPipelineMetric.java @@ -52,7 +52,12 @@ void postIngestBytes(long bytesProduced) { * Creates a serializable representation for these metrics. */ IngestStats.ByteStats createByteStats() { - return new IngestStats.ByteStats(this.bytesIngested.count(), this.bytesProduced.count()); + long bytesIngested = this.bytesIngested.count(); + long bytesProduced = this.bytesProduced.count(); + if (bytesIngested == 0L && bytesProduced == 0L) { + return IngestStats.ByteStats.IDENTITY; + } + return new IngestStats.ByteStats(bytesIngested, bytesProduced); } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index fa37d6147c6b3..03eef5f713854 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -1159,20 +1159,35 @@ static String getProcessorName(Processor processor) { if (processor instanceof ConditionalProcessor conditionalProcessor) { processor = conditionalProcessor.getInnerProcessor(); } - StringBuilder sb = new StringBuilder(5); - sb.append(processor.getType()); + String tag = processor.getTag(); + if (tag != null && tag.isEmpty()) { + tag = null; // it simplifies the rest of the logic slightly to coalesce to null + } + + String pipelineName = null; if (processor instanceof PipelineProcessor pipelineProcessor) { - String pipelineName = pipelineProcessor.getPipelineTemplate().newInstance(Map.of()).execute(); - sb.append(":"); - sb.append(pipelineName); + pipelineName = pipelineProcessor.getPipelineTemplate().newInstance(Map.of()).execute(); } - String tag = processor.getTag(); - if (tag != null && tag.isEmpty() == false) { - sb.append(":"); - sb.append(tag); + + // if there's a tag, OR if it's a pipeline processor, then the processor name is a compound thing, + // BUT if neither of those apply, then it's just the type -- so we can return the type itself without + // allocating a new String object + if (tag == null && pipelineName == null) { + return processor.getType(); + } else { + StringBuilder sb = new StringBuilder(5); + sb.append(processor.getType()); + if (pipelineName != null) { + sb.append(":"); + sb.append(pipelineName); + } + if (tag != null) { + sb.append(":"); + sb.append(tag); + } + return sb.toString(); } - return sb.toString(); } /** diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java index 39df4e53423a2..bc0fb26055e35 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Function; public record IngestStats(Stats totalStats, List pipelineStats, Map> processorStats) implements @@ -57,22 +58,33 @@ public record IngestStats(Stats totalStats, List pipelineStats, Ma * Read from a stream. */ public static IngestStats read(StreamInput in) throws IOException { - var stats = new Stats(in); + // while reading the processors, we're going to encounter identical name and type strings *repeatedly* + // it's advantageous to discard the endless copies of the same strings and canonical-ize them to keep our + // heap usage under control. note: this map is key to key, because of the limitations of the set interface. + final Map namesAndTypesCache = new HashMap<>(); + + var stats = readStats(in); var size = in.readVInt(); + if (stats == Stats.IDENTITY && size == 0) { + return IDENTITY; + } var pipelineStats = new ArrayList(size); var processorStats = Maps.>newMapWithExpectedSize(size); for (var i = 0; i < size; i++) { var pipelineId = in.readString(); - var pipelineStat = new Stats(in); - var byteStat = in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? new ByteStats(in) : new ByteStats(0, 0); + var pipelineStat = readStats(in); + var byteStat = in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? readByteStats(in) : ByteStats.IDENTITY; pipelineStats.add(new PipelineStat(pipelineId, pipelineStat, byteStat)); int processorsSize = in.readVInt(); var processorStatsPerPipeline = new ArrayList(processorsSize); for (var j = 0; j < processorsSize; j++) { var processorName = in.readString(); var processorType = in.readString(); - var processorStat = new Stats(in); + var processorStat = readStats(in); + // pass these name and type through the local names and types cache to canonical-ize them + processorName = namesAndTypesCache.computeIfAbsent(processorName, Function.identity()); + processorType = namesAndTypesCache.computeIfAbsent(processorType, Function.identity()); processorStatsPerPipeline.add(new ProcessorStat(processorName, processorType, processorStat)); } processorStats.put(pipelineId, Collections.unmodifiableList(processorStatsPerPipeline)); @@ -169,6 +181,21 @@ static Map> merge(Map> f return totalsPerPipelineProcessor; } + /** + * Read {@link Stats} from a stream. + */ + private static Stats readStats(StreamInput in) throws IOException { + long ingestCount = in.readVLong(); + long ingestTimeInMillis = in.readVLong(); + long ingestCurrent = in.readVLong(); + long ingestFailedCount = in.readVLong(); + if (ingestCount == 0 && ingestTimeInMillis == 0 && ingestCurrent == 0 && ingestFailedCount == 0) { + return Stats.IDENTITY; + } else { + return new Stats(ingestCount, ingestTimeInMillis, ingestCurrent, ingestFailedCount); + } + } + public record Stats(long ingestCount, long ingestTimeInMillis, long ingestCurrent, long ingestFailedCount) implements Writeable, @@ -176,13 +203,6 @@ public record Stats(long ingestCount, long ingestTimeInMillis, long ingestCurren public static final Stats IDENTITY = new Stats(0, 0, 0, 0); - /** - * Read from a stream. - */ - public Stats(StreamInput in) throws IOException { - this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong()); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(ingestCount); @@ -269,13 +289,21 @@ private static PipelineStat merge(PipelineStat first, PipelineStat second) { } } + static ByteStats readByteStats(StreamInput in) throws IOException { + long bytesIngested = in.readVLong(); + long bytesProduced = in.readVLong(); + if (bytesProduced == 0L && bytesIngested == 0L) { + return ByteStats.IDENTITY; + } + return new ByteStats(bytesIngested, bytesProduced); + } + /** * Container for ingested byte stats */ public record ByteStats(long bytesIngested, long bytesProduced) implements Writeable, ToXContentFragment { - public ByteStats(StreamInput in) throws IOException { - this(in.readVLong(), in.readVLong()); - } + + public static final ByteStats IDENTITY = new ByteStats(0L, 0L); @Override public void writeTo(StreamOutput out) throws IOException { @@ -299,6 +327,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } static ByteStats merge(ByteStats first, ByteStats second) { + if (first == IDENTITY) { + return second; + } else if (second == IDENTITY) { + return first; + } return new ByteStats((first.bytesIngested + second.bytesIngested), first.bytesProduced + second.bytesProduced); } } diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 9f3f3aaba62fc..44cd32b910342 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -46,7 +46,7 @@ public final class PipelineConfiguration implements SimpleDiffable builder.setConfig(parser.map()), + (parser, builder, aVoid) -> builder.setConfig(parser.mapOrdered()), new ParseField("config"), ObjectParser.ValueType.OBJECT ); diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java index 3e59814de0585..acf186faf20b2 100644 --- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java +++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java @@ -34,7 +34,7 @@ class CustomFieldHighlighter extends FieldHighlighter { private final Locale breakIteratorLocale; private final int noMatchSize; private String fieldValue; - private final Integer queryMaxAnalyzedOffset; + private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset; CustomFieldHighlighter( String field, @@ -47,7 +47,7 @@ class CustomFieldHighlighter extends FieldHighlighter { PassageFormatter passageFormatter, Comparator passageSortComparator, int noMatchSize, - Integer queryMaxAnalyzedOffset + QueryMaxAnalyzedOffset queryMaxAnalyzedOffset ) { super( field, @@ -113,7 +113,7 @@ protected Passage[] getSummaryPassagesNoHighlight(int maxPassages) { @Override protected Passage[] highlightOffsetsEnums(OffsetsEnum off) throws IOException { if (queryMaxAnalyzedOffset != null) { - off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset); + off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset.getNotNull()); } return super.highlightOffsetsEnums(off); } diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java index d1c7d0415ad15..59dffb73985ac 100644 --- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java +++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java @@ -66,7 +66,7 @@ public final class CustomUnifiedHighlighter extends UnifiedHighlighter { private final int noMatchSize; private final CustomFieldHighlighter fieldHighlighter; private final int maxAnalyzedOffset; - private final Integer queryMaxAnalyzedOffset; + private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset; /** * Creates a new instance of {@link CustomUnifiedHighlighter} @@ -94,7 +94,7 @@ public CustomUnifiedHighlighter( int noMatchSize, int maxPassages, int maxAnalyzedOffset, - Integer queryMaxAnalyzedOffset, + QueryMaxAnalyzedOffset queryMaxAnalyzedOffset, boolean requireFieldMatch, boolean weightMatchesEnabled ) { @@ -125,9 +125,9 @@ public Snippet[] highlightField(LeafReader reader, int docId, CheckedSupplier maxAnalyzedOffset) + if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset) && (getOffsetSource(field) == OffsetSource.ANALYSIS) - && (fieldValueLength > maxAnalyzedOffset))) { + && (fieldValueLength > maxAnalyzedOffset)) { throw new IllegalArgumentException( "The length [" + fieldValueLength diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java new file mode 100644 index 0000000000000..e74b11d4e1a91 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene.search.uhighlight; + +public class QueryMaxAnalyzedOffset { + private final int queryMaxAnalyzedOffset; + + private QueryMaxAnalyzedOffset(final int queryMaxAnalyzedOffset) { + // If we have a negative value, grab value for the actual maximum from the index. + this.queryMaxAnalyzedOffset = queryMaxAnalyzedOffset; + } + + public static QueryMaxAnalyzedOffset create(final Integer queryMaxAnalyzedOffset, final int indexMaxAnalyzedOffset) { + if (queryMaxAnalyzedOffset == null) { + return null; + } + return new QueryMaxAnalyzedOffset(queryMaxAnalyzedOffset < 0 ? indexMaxAnalyzedOffset : queryMaxAnalyzedOffset); + } + + public int getNotNull() { + return queryMaxAnalyzedOffset; + } +} diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 6f7a5e5ca585a..2568c2a1bb454 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -766,7 +766,7 @@ protected void validateNodeBeforeAcceptingRequests( * Writes a file to the logs dir containing the ports for the given transport type */ private void writePortsFile(String type, BoundTransportAddress boundAddress) { - Path tmpPortsFile = environment.logsFile().resolve(type + ".ports.tmp"); + Path tmpPortsFile = environment.logsDir().resolve(type + ".ports.tmp"); try (BufferedWriter writer = Files.newBufferedWriter(tmpPortsFile, Charset.forName("UTF-8"))) { for (TransportAddress address : boundAddress.boundAddresses()) { InetAddress inetAddress = InetAddress.getByName(address.getAddress()); @@ -775,7 +775,7 @@ private void writePortsFile(String type, BoundTransportAddress boundAddress) { } catch (IOException e) { throw new RuntimeException("Failed to write ports file", e); } - Path portsFile = environment.logsFile().resolve(type + ".ports"); + Path portsFile = environment.logsDir().resolve(type + ".ports"); try { Files.move(tmpPortsFile, portsFile, StandardCopyOption.ATOMIC_MOVE); } catch (IOException e) { diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index eb8675078274f..67dc89fd1c936 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -216,7 +216,6 @@ import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.upgrades.SystemIndexMigrationExecutor; import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -445,7 +444,7 @@ private Settings createEnvironment(Environment initialEnvironment, NodeServicePr ); } - if (initialEnvironment.dataFiles().length > 1) { + if (initialEnvironment.dataDirs().length > 1) { // NOTE: we use initialEnvironment here, but assertEquivalent below ensures the data paths do not change deprecationLogger.warn( DeprecationCategory.SETTINGS, @@ -466,10 +465,10 @@ private Settings createEnvironment(Environment initialEnvironment, NodeServicePr if (logger.isDebugEnabled()) { logger.debug( "using config [{}], data [{}], logs [{}], plugins [{}]", - initialEnvironment.configFile(), - Arrays.toString(initialEnvironment.dataFiles()), - initialEnvironment.logsFile(), - initialEnvironment.pluginsFile() + initialEnvironment.configDir(), + Arrays.toString(initialEnvironment.dataDirs()), + initialEnvironment.logsDir(), + initialEnvironment.pluginsDir() ); } @@ -486,7 +485,7 @@ private Settings createEnvironment(Environment initialEnvironment, NodeServicePr * Create the environment based on the finalized view of the settings. This is to ensure that components get the same setting * values, no matter they ask for them from. */ - environment = new Environment(settings, initialEnvironment.configFile()); + environment = new Environment(settings, initialEnvironment.configDir()); Environment.assertEquivalent(initialEnvironment, environment); modules.bindToInstance(Environment.class, environment); @@ -589,8 +588,7 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool, IndicesModule.getNamedWriteables().stream(), searchModule.getNamedWriteables().stream(), pluginsService.flatMap(Plugin::getNamedWriteables), - ClusterModule.getNamedWriteables().stream(), - SystemIndexMigrationExecutor.getNamedWriteables().stream() + ClusterModule.getNamedWriteables().stream() ).flatMap(Function.identity()).toList() ); xContentRegistry = new NamedXContentRegistry( @@ -600,7 +598,6 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool, searchModule.getNamedXContents().stream(), pluginsService.flatMap(Plugin::getNamedXContent), ClusterModule.getNamedXWriteables().stream(), - SystemIndexMigrationExecutor.getNamedXContentParsers().stream(), HealthNodeTaskExecutor.getNamedXContentParsers().stream() ).flatMap(Function.identity()).toList() ); @@ -1143,11 +1140,8 @@ public Map searchFields() { settingsModule, clusterService, threadPool, - systemIndices, featureService, - clusterModule.getIndexNameExpressionResolver(), - metadataUpdateSettingsService, - metadataCreateIndexService + clusterModule.getIndexNameExpressionResolver() ) ); @@ -1623,7 +1617,7 @@ private DiscoveryModule createDiscoveryModule( pluginsService.filterPlugins(DiscoveryPlugin.class).toList(), pluginsService.filterPlugins(ClusterCoordinationPlugin.class).toList(), allocationService, - environment.configFile(), + environment.configDir(), gatewayMetaState, rerouteService, fsHealthService, @@ -1644,21 +1638,10 @@ private Module loadPersistentTasksService( SettingsModule settingsModule, ClusterService clusterService, ThreadPool threadPool, - SystemIndices systemIndices, FeatureService featureService, - IndexNameExpressionResolver indexNameExpressionResolver, - MetadataUpdateSettingsService metadataUpdateSettingsService, - MetadataCreateIndexService metadataCreateIndexService + IndexNameExpressionResolver indexNameExpressionResolver ) { PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); - SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor( - client, - clusterService, - systemIndices, - metadataUpdateSettingsService, - metadataCreateIndexService, - settingsModule.getIndexScopedSettings() - ); HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create( clusterService, persistentTasksService, @@ -1666,7 +1649,7 @@ private Module loadPersistentTasksService( settingsModule.getSettings(), clusterService.getClusterSettings() ); - Stream> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor); + Stream> builtinTaskExecutors = Stream.of(healthNodeTaskExecutor); Stream> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) .map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule, indexNameExpressionResolver)) diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 8f2dc4e532ae0..230fe0e2acb0c 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -54,7 +54,7 @@ class NodeServiceProvider { PluginsService newPluginService(Environment initialEnvironment, PluginsLoader pluginsLoader) { // this creates a PluginsService with an empty list of classpath plugins - return new PluginsService(initialEnvironment.settings(), initialEnvironment.configFile(), pluginsLoader); + return new PluginsService(initialEnvironment.settings(), initialEnvironment.configDir(), pluginsLoader); } ScriptService newScriptService( diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 5623111dbbd2c..0c970f1e88c30 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -47,7 +47,7 @@ *

An implementation may also override {@link #prepareForIndicesMigration(ClusterService, Client, ActionListener)} and * {@link #indicesMigrationComplete(Map, ClusterService, Client, ActionListener)} in order to take special action before and after a * feature migration, which will temporarily block access to system indices. For example, a plugin might want to enter a safe mode and - * reject certain requests while the migration is in progress. See {@link org.elasticsearch.upgrades.SystemIndexMigrationExecutor} for + * reject certain requests while the migration is in progress. See org.elasticsearch.upgrades.SystemIndexMigrationExecutor for * more details. * *

After plugins are loaded, the {@link SystemIndices} class will provide the rest of the system with access to the feature's diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 713f7eacb2c16..645681445f55a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -29,6 +29,7 @@ import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.snapshots.SnapshotsService; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -377,6 +378,7 @@ private static boolean isIndexToUpdateAfterRemovingSnapshots( * @return map of index to index metadata blob id to delete */ public Map> indexMetaDataToRemoveAfterRemovingSnapshots(Collection snapshotIds) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SNAPSHOT); Iterator indicesForSnapshot = indicesToUpdateAfterRemovingSnapshot(snapshotIds); final Set allRemainingIdentifiers = indexMetaDataGenerations.lookup.entrySet() .stream() @@ -605,6 +607,11 @@ public int hashCode() { return Objects.hash(snapshotIds, snapshotsDetails, indices, indexSnapshots, shardGenerations, indexMetaDataGenerations); } + @Override + public String toString() { + return Strings.format("RepositoryData[uuid=%s,gen=%s]", uuid, genId); + } + /** * Resolve the index name to the index id specific to the repository, * throwing an exception if the index could not be resolved. diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 240d41ea589a7..9e1f528f1f37f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -1129,14 +1129,20 @@ private void runWithUniqueShardMetadataNaming(ActionListener rep ); }) - .andThen((l, newRepositoryData) -> { - l.onResponse(newRepositoryData); - // Once we have updated the repository, run the unreferenced blobs cleanup in parallel to shard-level snapshot deletion - try (var refs = new RefCountingRunnable(onCompletion)) { - cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener()); - cleanupUnlinkedShardLevelBlobs(refs.acquireListener()); + .andThen( + // writeIndexGen finishes on master-service thread so must fork here. + snapshotExecutor, + threadPool.getThreadContext(), + (l, newRepositoryData) -> { + l.onResponse(newRepositoryData); + // Once we have updated the repository, run the unreferenced blobs cleanup in parallel to shard-level snapshot + // deletion + try (var refs = new RefCountingRunnable(onCompletion)) { + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener()); + cleanupUnlinkedShardLevelBlobs(refs.acquireListener()); + } } - }) + ) .addListener(repositoryDataUpdateListener); } diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java index a97f2d4248719..5b1fd843b6a4a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java @@ -92,13 +92,13 @@ public FsRepository( ); throw new RepositoryException(metadata.name(), "missing location"); } - Path locationFile = environment.resolveRepoFile(location); + Path locationFile = environment.resolveRepoDir(location); if (locationFile == null) { - if (environment.repoFiles().length > 0) { + if (environment.repoDirs().length > 0) { logger.warn( "The specified location [{}] doesn't start with any " + "repository paths specified by the path.repo setting: [{}] ", location, - environment.repoFiles() + environment.repoDirs() ); throw new RepositoryException( metadata.name(), @@ -127,7 +127,7 @@ public FsRepository( @Override protected BlobStore createBlobStore() throws Exception { final String location = REPOSITORIES_LOCATION_SETTING.get(getMetadata().settings()); - final Path locationFile = environment.resolveRepoFile(location); + final Path locationFile = environment.resolveRepoDir(location); return new FsBlobStore(bufferSize, locationFile, isReadOnly()); } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index 811b59465ce76..85874443477e8 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -24,8 +24,13 @@ import java.io.BufferedInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; import static org.elasticsearch.reservedstate.service.ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION; import static org.elasticsearch.reservedstate.service.ReservedStateVersionCheck.HIGHER_VERSION_ONLY; @@ -59,7 +64,7 @@ public class FileSettingsService extends MasterNodeFileWatchingService implement * @param environment we need the environment to pull the location of the config and operator directories */ public FileSettingsService(ClusterService clusterService, ReservedClusterStateService stateService, Environment environment) { - super(clusterService, environment.configFile().toAbsolutePath().resolve(OPERATOR_DIRECTORY).resolve(SETTINGS_FILE_NAME)); + super(clusterService, environment.configDir().toAbsolutePath().resolve(OPERATOR_DIRECTORY).resolve(SETTINGS_FILE_NAME)); this.stateService = stateService; } @@ -84,7 +89,7 @@ public void handleSnapshotRestore(ClusterState clusterState, Metadata.Builder md // since we don't know the current operator configuration, e.g. file settings could be disabled // on the target cluster. If file settings exist and the cluster state has lost it's reserved // state for the "file_settings" namespace, we touch our file settings file to cause it to re-process the file. - if (watching() && Files.exists(watchedFile())) { + if (watching() && filesExists(watchedFile())) { if (fileSettingsMetadata != null) { ReservedStateMetadata withResetVersion = new ReservedStateMetadata.Builder(fileSettingsMetadata).version(0L).build(); mdBuilder.put(withResetVersion); @@ -134,7 +139,7 @@ protected void processFileOnServiceStart() throws IOException, ExecutionExceptio private void processFileChanges(ReservedStateVersionCheck versionCheck) throws IOException, InterruptedException, ExecutionException { PlainActionFuture completion = new PlainActionFuture<>(); try ( - var fis = Files.newInputStream(watchedFile()); + var fis = filesNewInputStream(watchedFile()); var bis = new BufferedInputStream(fis); var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis) ) { @@ -158,4 +163,37 @@ private static void completeProcessing(Exception e, PlainActionFuture comp completion.onResponse(null); } } + + // the following methods are a workaround to ensure exclusive access for files + // required by child watchers; this is required because we only check the caller's module + // not the entire stack + @Override + protected boolean filesExists(Path path) { + return Files.exists(path); + } + + @Override + protected boolean filesIsDirectory(Path path) { + return Files.isDirectory(path); + } + + @Override + protected A filesReadAttributes(Path path, Class clazz) throws IOException { + return Files.readAttributes(path, clazz); + } + + @Override + protected Stream filesList(Path dir) throws IOException { + return Files.list(dir); + } + + @Override + protected Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException { + return Files.setLastModifiedTime(path, time); + } + + @Override + protected InputStream filesNewInputStream(Path path) throws IOException { + return Files.newInputStream(path); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java index 33b3ef35671e3..e4e8378e4355e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java @@ -18,14 +18,14 @@ import org.elasticsearch.client.internal.FilterClient; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.Nullable; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -112,12 +112,14 @@ private void cancelTask(TaskId taskId) { private class CloseListener implements ActionListener { private final AtomicReference channel = new AtomicReference<>(); - private final Set tasks = new HashSet<>(); + + @Nullable // if already drained + private Set tasks = new HashSet<>(); CloseListener() {} synchronized int getNumTasks() { - return tasks.size(); + return tasks == null ? 0 : tasks.size(); } void maybeRegisterChannel(HttpChannel httpChannel) { @@ -130,16 +132,23 @@ void maybeRegisterChannel(HttpChannel httpChannel) { } } - synchronized void registerTask(TaskHolder taskHolder, TaskId taskId) { - taskHolder.taskId = taskId; - if (taskHolder.completed == false) { - this.tasks.add(taskId); + void registerTask(TaskHolder taskHolder, TaskId taskId) { + synchronized (this) { + taskHolder.taskId = taskId; + if (tasks != null) { + if (taskHolder.completed == false) { + tasks.add(taskId); + } + return; + } } + // else tasks == null so the channel is already closed + cancelTask(taskId); } synchronized void unregisterTask(TaskHolder taskHolder) { - if (taskHolder.taskId != null) { - this.tasks.remove(taskHolder.taskId); + if (taskHolder.taskId != null && tasks != null) { + tasks.remove(taskHolder.taskId); } taskHolder.completed = true; } @@ -149,18 +158,20 @@ public void onResponse(Void aVoid) { final HttpChannel httpChannel = channel.get(); assert httpChannel != null : "channel not registered"; // when the channel gets closed it won't be reused: we can remove it from the map and forget about it. - CloseListener closeListener = httpChannels.remove(httpChannel); - assert closeListener != null : "channel not found in the map of tracked channels"; - final List toCancel; - synchronized (this) { - toCancel = new ArrayList<>(tasks); - tasks.clear(); - } - for (TaskId taskId : toCancel) { + final CloseListener closeListener = httpChannels.remove(httpChannel); + assert closeListener != null : "channel not found in the map of tracked channels: " + httpChannel; + assert closeListener == CloseListener.this : "channel had a different CloseListener registered: " + httpChannel; + for (final var taskId : drainTasks()) { cancelTask(taskId); } } + private synchronized Collection drainTasks() { + final var drained = tasks; + tasks = null; + return drained; + } + @Override public void onFailure(Exception e) { onResponse(null); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java index 900a352d42f30..fa2de167ea17e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java @@ -26,5 +26,17 @@ public class CreateIndexCapabilities { */ private static final String LOOKUP_INDEX_MODE_CAPABILITY = "lookup_index_mode"; - public static Set CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY, LOOKUP_INDEX_MODE_CAPABILITY); + private static final String NESTED_DENSE_VECTOR_SYNTHETIC_TEST = "nested_dense_vector_synthetic_test"; + + private static final String POORLY_FORMATTED_BAD_REQUEST = "poorly_formatted_bad_request"; + + private static final String HUNSPELL_DICT_400 = "hunspell_dict_400"; + + public static final Set CAPABILITIES = Set.of( + LOGSDB_INDEX_MODE_CAPABILITY, + LOOKUP_INDEX_MODE_CAPABILITY, + NESTED_DENSE_VECTOR_SYNTHETIC_TEST, + POORLY_FORMATTED_BAD_REQUEST, + HUNSPELL_DICT_400 + ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 18f7cc1222d5f..83abc0555b0b8 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -50,9 +50,9 @@ public String getName() { @Override public Set supportedCapabilities() { if (DataStream.isFailureStoreFeatureFlagEnabled()) { - return Set.of("lazy-rollover-failure-store", "index-expression-selectors"); + return Set.of("return-404-on-missing-target", "lazy-rollover-failure-store", "index-expression-selectors"); } else { - return Set.of(); + return Set.of("return-404-on-missing-target"); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 5fb8759374865..e182177e8508b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -43,6 +43,8 @@ private SearchCapabilities() {} private static final String OPTIMIZED_SCALAR_QUANTIZATION_BBQ = "optimized_scalar_quantization_bbq"; private static final String KNN_QUANTIZED_VECTOR_RESCORE_OVERSAMPLE = "knn_quantized_vector_rescore_oversample"; + private static final String HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT = "highlight_max_analyzed_offset_default"; + public static final Set CAPABILITIES; static { HashSet capabilities = new HashSet<>(); @@ -58,6 +60,7 @@ private SearchCapabilities() {} capabilities.add(K_DEFAULT_TO_SIZE); capabilities.add(KQL_QUERY_SUPPORTED); capabilities.add(RRF_WINDOW_SIZE_SUPPORT_DEPRECATED); + capabilities.add(HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT); CAPABILITIES = Set.copyOf(capabilities); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java index f24052ef7e3a9..e085eb50ffb9f 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.script.ScriptContextStats.Fields.CACHE_EVICTIONS_HISTORY; import static org.elasticsearch.script.ScriptContextStats.Fields.COMPILATIONS_HISTORY; import static org.elasticsearch.script.ScriptStats.Fields.CACHE_EVICTIONS; import static org.elasticsearch.script.ScriptStats.Fields.COMPILATIONS; @@ -199,7 +200,7 @@ public Iterator toXContentChunked(ToXContent.Params params ob.xContentObject(COMPILATIONS_HISTORY, compilationsHistory); } if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) { - ob.xContentObject(COMPILATIONS_HISTORY, cacheEvictionsHistory); + ob.xContentObject(CACHE_EVICTIONS_HISTORY, cacheEvictionsHistory); } ob.array(CONTEXTS, contextStats.iterator()); }); diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index e0c0e025ee5f6..cc2ee78687737 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -158,6 +158,7 @@ import java.util.function.Supplier; import static org.elasticsearch.TransportVersions.ERROR_TRACE_IN_TRANSPORT_HEADER; +import static org.elasticsearch.common.Strings.format; import static org.elasticsearch.core.TimeValue.timeValueHours; import static org.elasticsearch.core.TimeValue.timeValueMillis; import static org.elasticsearch.core.TimeValue.timeValueMinutes; @@ -529,12 +530,18 @@ protected void doClose() { * @param the type of the response * @param listener the action listener to be wrapped * @param version channel version of the request + * @param nodeId id of the current node + * @param shardId id of the shard being searched + * @param taskId id of the task being executed * @param threadPool with context where to write the new header * @return the wrapped action listener */ static ActionListener maybeWrapListenerForStackTrace( ActionListener listener, TransportVersion version, + String nodeId, + ShardId shardId, + long taskId, ThreadPool threadPool ) { boolean header = true; @@ -543,6 +550,18 @@ static ActionListener maybeWrapListenerForStackTrace( } if (header == false) { return listener.delegateResponse((l, e) -> { + org.apache.logging.log4j.util.Supplier messageSupplier = () -> format( + "[%s]%s: failed to execute search request for task [%d]", + nodeId, + shardId, + taskId + ); + // Keep this logic aligned with that of SUPPRESSED_ERROR_LOGGER in RestResponse + if (ExceptionsHelper.status(e).getStatus() < 500 || ExceptionsHelper.isNodeOrShardUnavailableTypeException(e)) { + logger.debug(messageSupplier, e); + } else { + logger.warn(messageSupplier, e); + } ExceptionsHelper.unwrapCausesAndSuppressed(e, err -> { err.setStackTrace(EMPTY_STACK_TRACE_ARRAY); return false; @@ -554,7 +573,14 @@ static ActionListener maybeWrapListenerForStackTrace( } public void executeDfsPhase(ShardSearchRequest request, SearchShardTask task, ActionListener listener) { - listener = maybeWrapListenerForStackTrace(listener, request.getChannelVersion(), threadPool); + listener = maybeWrapListenerForStackTrace( + listener, + request.getChannelVersion(), + clusterService.localNode().getId(), + request.shardId(), + task.getId(), + threadPool + ); final IndexShard shard = getShard(request); rewriteAndFetchShardRequest(shard, request, listener.delegateFailure((l, rewritten) -> { // fork the execution in the search thread pool @@ -592,7 +618,14 @@ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final Sea } public void executeQueryPhase(ShardSearchRequest request, SearchShardTask task, ActionListener listener) { - ActionListener finalListener = maybeWrapListenerForStackTrace(listener, request.getChannelVersion(), threadPool); + ActionListener finalListener = maybeWrapListenerForStackTrace( + listener, + request.getChannelVersion(), + clusterService.localNode().getId(), + request.shardId(), + task.getId(), + threadPool + ); assert request.canReturnNullResponseIfMatchNoDocs() == false || request.numberOfShards() > 1 : "empty responses require more than one shard"; final IndexShard shard = getShard(request); @@ -785,9 +818,16 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh } public void executeRankFeaturePhase(RankFeatureShardRequest request, SearchShardTask task, ActionListener listener) { - listener = maybeWrapListenerForStackTrace(listener, request.getShardSearchRequest().getChannelVersion(), threadPool); final ReaderContext readerContext = findReaderContext(request.contextId(), request); final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest()); + listener = maybeWrapListenerForStackTrace( + listener, + shardSearchRequest.getChannelVersion(), + clusterService.localNode().getId(), + shardSearchRequest.shardId(), + task.getId(), + threadPool + ); final Releasable markAsUsed = readerContext.markAsUsed(getKeepAlive(shardSearchRequest)); runAsync(getExecutor(readerContext.indexShard()), () -> { try (SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, ResultsType.RANK_FEATURE, false)) { @@ -832,8 +872,15 @@ public void executeQueryPhase( ActionListener listener, TransportVersion version ) { - listener = maybeWrapListenerForStackTrace(listener, version, threadPool); final LegacyReaderContext readerContext = (LegacyReaderContext) findReaderContext(request.contextId(), request); + listener = maybeWrapListenerForStackTrace( + listener, + version, + clusterService.localNode().getId(), + readerContext.indexShard().shardId(), + task.getId(), + threadPool + ); final Releasable markAsUsed; try { markAsUsed = readerContext.markAsUsed(getScrollKeepAlive(request.scroll())); @@ -874,9 +921,16 @@ public void executeQueryPhase( ActionListener listener, TransportVersion version ) { - listener = maybeWrapListenerForStackTrace(listener, version, threadPool); final ReaderContext readerContext = findReaderContext(request.contextId(), request.shardSearchRequest()); final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest()); + listener = maybeWrapListenerForStackTrace( + listener, + version, + clusterService.localNode().getId(), + shardSearchRequest.shardId(), + task.getId(), + threadPool + ); final Releasable markAsUsed = readerContext.markAsUsed(getKeepAlive(shardSearchRequest)); rewriteAndFetchShardRequest(readerContext.indexShard(), shardSearchRequest, listener.delegateFailure((l, rewritten) -> { // fork the execution in the search thread pool diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java index ef21e4103fd88..f763ac8f795ff 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java @@ -211,9 +211,9 @@ private List reducePipelineBuckets(AggregationReduceContext reduceContext, Pi List reducedBuckets = new ArrayList<>(); for (B bucket : getBuckets()) { List aggs = new ArrayList<>(); - for (Aggregation agg : bucket.getAggregations()) { + for (InternalAggregation agg : bucket.getAggregations()) { PipelineTree subTree = pipelineTree.subTree(agg.getName()); - aggs.add(((InternalAggregation) agg).reducePipelines((InternalAggregation) agg, reduceContext, subTree)); + aggs.add(agg.reducePipelines(agg, reduceContext, subTree)); } reducedBuckets.add(createBucket(InternalAggregations.from(aggs), bucket)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InvalidAggregationPathException.java b/server/src/main/java/org/elasticsearch/search/aggregations/InvalidAggregationPathException.java index 739dc58743332..34fcf58e43bd3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InvalidAggregationPathException.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InvalidAggregationPathException.java @@ -20,10 +20,6 @@ public InvalidAggregationPathException(String msg) { super(msg); } - public InvalidAggregationPathException(String msg, Throwable cause) { - super(msg, cause); - } - public InvalidAggregationPathException(StreamInput in) throws IOException { super(in); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java index 592f7b4887598..e85d01930807c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; import java.util.AbstractList; @@ -163,6 +164,10 @@ protected void prepareSubAggs(LongArray ordsToCollect) throws IOException {} * array of ordinals */ protected final IntFunction buildSubAggsForBuckets(LongArray bucketOrdsToCollect) throws IOException { + if (context.isCancelled()) { + throw new TaskCancelledException("not building sub-aggregations due to task cancellation"); + } + prepareSubAggs(bucketOrdsToCollect); InternalAggregation[][] aggregations = new InternalAggregation[subAggregators.length][]; for (int i = 0; i < subAggregators.length; i++) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java index 6bf456c71929a..b97176b50ba7f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -461,7 +461,7 @@ protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCt // Visit documents sorted by the leading source of the composite definition and terminates // when the leading source value is guaranteed to be greater than the lowest composite bucket // in the queue. - DocIdSet docIdSet = sortedDocsProducer.processLeaf(topLevelQuery(), queue, aggCtx.getLeafReaderContext(), fillDocIdSet); + DocIdSet docIdSet = sortedDocsProducer.processLeaf(queue, aggCtx.getLeafReaderContext(), fillDocIdSet); if (fillDocIdSet) { entries.add(new Entry(aggCtx, docIdSet)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java index 7c920abfe2451..e88c9724edba1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/PointsSortedDocsProducer.java @@ -13,7 +13,6 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.Query; import org.apache.lucene.util.DocIdSetBuilder; import java.io.IOException; @@ -36,8 +35,7 @@ class PointsSortedDocsProducer extends SortedDocsProducer { } @Override - DocIdSet processLeaf(Query query, CompositeValuesCollectorQueue queue, LeafReaderContext context, boolean fillDocIdSet) - throws IOException { + DocIdSet processLeaf(CompositeValuesCollectorQueue queue, LeafReaderContext context, boolean fillDocIdSet) throws IOException { final PointValues values = context.reader().getPointValues(field); if (values == null) { // no value for the field diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/SortedDocsProducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/SortedDocsProducer.java index 4503758c55b04..2d1b628482d45 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/SortedDocsProducer.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/SortedDocsProducer.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Query; import org.apache.lucene.util.Bits; import org.apache.lucene.util.DocIdSetBuilder; import org.elasticsearch.core.Nullable; @@ -99,6 +98,5 @@ public void collect(int doc, long bucket) throws IOException { * Returns the {@link DocIdSet} of the documents that contain a top composite bucket in this leaf or * {@link DocIdSet#EMPTY} if fillDocIdSet is false. */ - abstract DocIdSet processLeaf(Query query, CompositeValuesCollectorQueue queue, LeafReaderContext context, boolean fillDocIdSet) - throws IOException; + abstract DocIdSet processLeaf(CompositeValuesCollectorQueue queue, LeafReaderContext context, boolean fillDocIdSet) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsSortedDocsProducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsSortedDocsProducer.java index e2aaba06a19ec..3b62cb8f57d8b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsSortedDocsProducer.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsSortedDocsProducer.java @@ -14,7 +14,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.DocIdSetBuilder; @@ -29,8 +28,7 @@ class TermsSortedDocsProducer extends SortedDocsProducer { } @Override - DocIdSet processLeaf(Query query, CompositeValuesCollectorQueue queue, LeafReaderContext context, boolean fillDocIdSet) - throws IOException { + DocIdSet processLeaf(CompositeValuesCollectorQueue queue, LeafReaderContext context, boolean fillDocIdSet) throws IOException { final Terms terms = context.reader().terms(field); if (terms == null) { // no value for the field diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java index 7291a099dd7f7..9994a2bca08bf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java @@ -25,15 +25,7 @@ public class InternalDateRange extends InternalRange aggregations, - boolean keyed, - DocValueFormat formatter - ) { + public Bucket(String key, double from, double to, long docCount, List aggregations, DocValueFormat formatter) { super(key, from, to, docCount, InternalAggregations.from(aggregations), formatter); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BytesKeyedBucketOrds.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BytesKeyedBucketOrds.java index 17982043e8e20..b65b0e1ec010a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BytesKeyedBucketOrds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BytesKeyedBucketOrds.java @@ -75,24 +75,6 @@ public interface BucketOrdsEnum { * Read the current value. */ void readValue(BytesRef dest); - - /** - * An {@linkplain BucketOrdsEnum} that is empty. - */ - BucketOrdsEnum EMPTY = new BucketOrdsEnum() { - @Override - public boolean next() { - return false; - } - - @Override - public long ord() { - return 0; - } - - @Override - public void readValue(BytesRef dest) {} - }; } /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java index 5108793b8a809..9db9a41016621 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java @@ -72,7 +72,7 @@ protected Bucket(long subsetDf, long supersetDf, InternalAggregations aggregatio /** * Read from a stream. */ - protected Bucket(StreamInput in, DocValueFormat format) { + protected Bucket(DocValueFormat format) { this.format = format; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantLongTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantLongTerms.java index 17ea290b7aaaf..807514b30ab5b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantLongTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantLongTerms.java @@ -37,7 +37,7 @@ public Bucket(long subsetDf, long supersetDf, long term, InternalAggregations ag } Bucket(StreamInput in, DocValueFormat format) throws IOException { - super(in, format); + super(format); subsetDf = in.readVLong(); supersetDf = in.readVLong(); term = in.readLong(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantStringTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantStringTerms.java index b255f17d2843b..d55228304666f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantStringTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantStringTerms.java @@ -48,7 +48,7 @@ public Bucket( * Read from a stream. */ public Bucket(StreamInput in, DocValueFormat format) throws IOException { - super(in, format); + super(format); termBytes = in.readBytesRef(); subsetDf = in.readVLong(); supersetDf = in.readVLong(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java index 039bd0dd67592..1b8cb51c4460b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHDRPercentilesAggregator.java @@ -108,8 +108,7 @@ protected DoubleHistogram getState(long bucketOrd) { if (bucketOrd >= states.size()) { return null; } - final DoubleHistogram state = states.get(bucketOrd); - return state; + return states.get(bucketOrd); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java index 00d7890e4710f..2605fc1c09361 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java @@ -44,9 +44,6 @@ public AbstractHyperLogLogPlusPlus(int precision) { /** Get HyperLogLog algorithm */ protected abstract AbstractHyperLogLog.RunLenIterator getHyperLogLog(long bucketOrd); - /** Get the number of data structures */ - public abstract long maxOrd(); - /** Collect a value in the given bucket */ public abstract void collect(long bucketOrd, long hash); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractLinearCounting.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractLinearCounting.java index 1f1cbd0b34a67..908fb4bb0a2e3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractLinearCounting.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractLinearCounting.java @@ -39,11 +39,6 @@ public AbstractLinearCounting(int precision) { */ protected abstract int size(long bucketOrd); - /** - * return the current values in the counter. - */ - protected abstract HashesIterator values(long bucketOrd); - public int collect(long bucketOrd, long hash) { final int k = encodeHash(hash, p); return addEncoded(bucketOrd, k); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStats.java index 5edcf745c418d..08e9de383691b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStats.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStats.java @@ -68,11 +68,6 @@ public interface ExtendedStats extends Stats { */ String getStdDeviationSamplingAsString(); - /** - * The upper or lower bounds of stdDev of the collected values as a String. - */ - String getStdDeviationBoundAsString(Bounds bound); - /** * The sum of the squares of the collected values as a String. */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlus.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlus.java index 5af6a50a8c4a6..16dfbdada4b0a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlus.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlus.java @@ -87,7 +87,6 @@ public HyperLogLogPlusPlus(int precision, BigArrays bigArrays, long initialBucke this.algorithm = algorithm; } - @Override public long maxOrd() { return hll.maxOrd(); } @@ -322,8 +321,7 @@ protected int size(long bucketOrd) { return size; } - @Override - protected HashesIterator values(long bucketOrd) { + private HashesIterator values(long bucketOrd) { // Make a fresh BytesRef for reading scratch work because this method can be called on many threads return new LinearCountingIterator(this, new BytesRef(), bucketOrd); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java index 1736b5ea7656d..8b1dcfb8a2f85 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java @@ -36,11 +36,6 @@ final class HyperLogLogPlusPlusSparse extends AbstractHyperLogLogPlusPlus implem this.lc = new LinearCounting(precision, bigArrays, initialBuckets); } - @Override - public long maxOrd() { - return lc.sizes.size(); - } - /** Needs to be called before adding elements into a bucket */ protected void ensureCapacity(long bucketOrd, long size) { lc.ensureCapacity(bucketOrd, size); @@ -135,8 +130,7 @@ protected int size(long bucketOrd) { return size; } - @Override - protected HashesIterator values(long bucketOrd) { + private HashesIterator values(long bucketOrd) { return new LinearCountingIterator(values.get(bucketOrd), size(bucketOrd)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalBounds.java index c3a106bd9af41..8a128b77a7300 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalBounds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalBounds.java @@ -73,9 +73,8 @@ public Object getProperty(List path) { }; } else if (path.size() == 2) { BoundingBox bbox = resolveBoundingBox(); - T cornerPoint = null; String cornerString = path.get(0); - cornerPoint = switch (cornerString) { + T cornerPoint = switch (cornerString) { case "top_left" -> bbox.topLeft(); case "bottom_right" -> bbox.bottomRight(); default -> throw new IllegalArgumentException("Found unknown path element [" + cornerString + "] in [" + getName() + "]"); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java index 7965211e24683..c6f4adc735c0c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java @@ -245,8 +245,7 @@ public String getStdDeviationSamplingAsString() { return valueAsString(Metrics.std_deviation_sampling.name()); } - @Override - public String getStdDeviationBoundAsString(Bounds bound) { + private String getStdDeviationBoundAsString(Bounds bound) { return switch (bound) { case UPPER -> valueAsString(Metrics.std_upper.name()); case LOWER -> valueAsString(Metrics.std_lower.name()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java index 48adad3cee618..e537c7348da6f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java @@ -90,15 +90,6 @@ protected MultiValue(StreamInput in) throws IOException { super(in); } - /** - * Read from a stream. - * - * @param readFormat whether to read the "format" field - */ - protected MultiValue(StreamInput in, boolean readFormat) throws IOException { - super(in, readFormat); - } - public abstract double value(String name); public String valueAsString(String name) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index ac37b287736aa..2ec30b411928a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -662,7 +662,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa factory.fetchSource(FetchSourceContext.fromXContent(parser)); } else if (SearchSourceBuilder.SCRIPT_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { List scriptFields = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { String scriptFieldName = parser.currentName(); token = parser.nextToken(); if (token == XContentParser.Token.START_OBJECT) { @@ -740,12 +740,12 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa parser ); } else if (SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { FieldAndFormat ff = FieldAndFormat.fromXContent(parser); factory.docValueField(ff.field, ff.format); } } else if (SearchSourceBuilder.FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { FieldAndFormat ff = FieldAndFormat.fromXContent(parser); factory.fetchField(ff); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java index 32d0ae6596c85..3f7da293dfa14 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java @@ -68,8 +68,7 @@ public void writeTo(StreamOutput out) throws IOException { */ @Override public final PipelineAggregator create() { - PipelineAggregator aggregator = createInternal(this.metadata); - return aggregator; + return createInternal(this.metadata); } @SuppressWarnings("unchecked") diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsParser.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsParser.java index e372e328ec88e..8765657e8a4d3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsParser.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsParser.java @@ -56,7 +56,7 @@ public final BucketMetricsPipelineAggregationBuilder parse(String pipelineAgg } else if (token == XContentParser.Token.START_ARRAY) { if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) { List paths = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { String path = parser.text(); paths.add(path); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java index 802aef5be68f3..1213b1a71761d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java @@ -28,8 +28,8 @@ public class InternalBucketMetricValue extends InternalNumericMetricsAggregation public static final String NAME = "bucket_metric_value"; static final ParseField KEYS_FIELD = new ParseField("keys"); - private double value; - private String[] keys; + private final double value; + private final String[] keys; public InternalBucketMetricValue(String name, String[] keys, double value, DocValueFormat formatter, Map metadata) { super(name, formatter, metadata); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java index 667e34d85b791..beb125608cbe4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java @@ -26,9 +26,9 @@ import java.util.Objects; public class InternalPercentilesBucket extends InternalNumericMetricsAggregation.MultiValue implements PercentilesBucket { - private double[] percentiles; - private double[] percents; - private boolean keyed = true; + private final double[] percentiles; + private final double[] percents; + private final boolean keyed; private final transient Map percentileLookups = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregator.java index 8337d644c9a9b..86807e9772a2b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregator.java @@ -21,7 +21,7 @@ public class PercentilesBucketPipelineAggregator extends BucketMetricsPipelineAggregator { private final double[] percents; - private boolean keyed = true; + private final boolean keyed; private List data; PercentilesBucketPipelineAggregator( diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java index 2537d79a40bf5..03b4867f6036b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.DocValueFormat; @@ -169,11 +170,11 @@ public static SerialDiffPipelineAggregationBuilder parse(String reducerName, XCo } else if (token == XContentParser.Token.START_ARRAY) { if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) { List paths = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { String path = parser.text(); paths.add(path); } - bucketsPaths = paths.toArray(new String[paths.size()]); + bucketsPaths = paths.toArray(Strings.EMPTY_ARRAY); } else { throw new ParsingException( parser.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index 4f939ea294e48..f26abecfa4ca0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.DocCountFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.Rewriteable; @@ -309,14 +308,6 @@ public final AggregationUsageService getUsageService() { public abstract Set sourcePath(String fullName); - /** - * Returns the MappingLookup for the index, if one is initialized. - */ - @Nullable - public MappingLookup getMappingLookup() { - return null; - } - /** * Does this index have a {@code _doc_count} field in any segment? */ @@ -621,11 +612,6 @@ public Set sourcePath(String fullName) { return context.sourcePath(fullName); } - @Override - public MappingLookup getMappingLookup() { - return context.getMappingLookup(); - } - @Override public void close() { /* diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java index 31adf423d74c9..23ccf1d940849 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationUsageService.java @@ -19,8 +19,6 @@ public class AggregationUsageService implements ReportingService { private static final String ES_SEARCH_QUERY_AGGREGATIONS_TOTAL_COUNT = "es.search.query.aggregations.total"; - private final String AGGREGATION_NAME_KEY = "aggregation_name"; - private final String VALUES_SOURCE_KEY = "values_source"; private final LongCounter aggregationsUsageCounter; private final Map> aggs; private final AggregationInfo info; @@ -83,6 +81,8 @@ public void incAggregationUsage(String aggregationName, String valuesSourceType) } assert valuesSourceMap != null : "Unknown aggregation [" + aggregationName + "][" + valuesSourceType + "]"; // tests will have a no-op implementation here + String VALUES_SOURCE_KEY = "values_source"; + String AGGREGATION_NAME_KEY = "aggregation_name"; aggregationsUsageCounter.incrementBy(1, Map.of(AGGREGATION_NAME_KEY, aggregationName, VALUES_SOURCE_KEY, valuesSourceType)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java index ffdbfffbce9e9..3e74d163b0d9f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java @@ -57,10 +57,9 @@ public static void declareCommon( * @param timezoneAware - allows specifying timezone * @param filterable - allows specifying filters on the values * @param heterogeneous - allows specifying value-source specific format and user value type hint - * @param - values source type * @param - parser context */ - public static void declareField( + public static void declareField( String fieldName, AbstractObjectParser, T> objectParser, boolean scriptable, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java index 742d366efa7a3..2e46a1512a7d1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java @@ -263,11 +263,7 @@ private boolean isInvalidDoc(int docId) throws IOException { // true if the TSID ord has changed since the last time we checked boolean shouldPop() throws IOException { - if (tsidOrd != tsids.ordValue()) { - return true; - } else { - return false; - } + return tsidOrd != tsids.ordValue(); } } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java index e14177adba467..14e1a66843e17 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java @@ -568,13 +568,12 @@ public Integer phraseLimit() { } /** - * Set to a non-negative value which represents the max offset used to analyze - * the field thus avoiding exceptions if the field exceeds this limit. + * "maxAnalyzedOffset" might be non-negative int, null (unknown), or a negative int (defaulting to index analyzed offset). */ @SuppressWarnings("unchecked") public HB maxAnalyzedOffset(Integer maxAnalyzedOffset) { - if (maxAnalyzedOffset != null && maxAnalyzedOffset <= 0) { - throw new IllegalArgumentException("[" + MAX_ANALYZED_OFFSET_FIELD + "] must be a positive integer"); + if (maxAnalyzedOffset != null && (maxAnalyzedOffset < -1 || maxAnalyzedOffset == 0)) { + throw new IllegalArgumentException("[" + MAX_ANALYZED_OFFSET_FIELD + "] must be a positive integer, or -1"); } this.maxAnalyzedOffset = maxAnalyzedOffset; return (HB) this; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java index 954505f5f3625..af5636a11c29b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java @@ -31,6 +31,7 @@ import org.elasticsearch.lucene.search.uhighlight.BoundedBreakIteratorScanner; import org.elasticsearch.lucene.search.uhighlight.CustomPassageFormatter; import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter; +import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset; import org.elasticsearch.lucene.search.uhighlight.Snippet; import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; @@ -121,7 +122,10 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) { int maxAnalyzedOffset = indexSettings.getHighlightMaxAnalyzedOffset(); boolean weightMatchesEnabled = indexSettings.isWeightMatchesEnabled(); int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments(); - Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset(); + QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create( + fieldContext.field.fieldOptions().maxAnalyzedOffset(), + maxAnalyzedOffset + ); Analyzer analyzer = wrapAnalyzer( fieldContext.context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER), queryMaxAnalyzedOffset @@ -171,7 +175,7 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) { fieldContext.field.fieldOptions().noMatchSize(), highlighterNumberOfFragments, maxAnalyzedOffset, - fieldContext.field.fieldOptions().maxAnalyzedOffset(), + queryMaxAnalyzedOffset, fieldContext.field.fieldOptions().requireFieldMatch(), weightMatchesEnabled ); @@ -186,9 +190,9 @@ protected PassageFormatter getPassageFormatter(SearchHighlightContext.Field fiel ); } - protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) { + protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) { if (maxAnalyzedOffset != null) { - analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset); + analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull()); } return analyzer; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java index 3dd3dd1b42c8c..e1c09e925c1b4 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset; import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; @@ -107,7 +108,10 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc ArrayList fragsList = new ArrayList<>(); List textsToHighlight; final int maxAnalyzedOffset = context.getSearchExecutionContext().getIndexSettings().getHighlightMaxAnalyzedOffset(); - Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset(); + QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create( + fieldContext.field.fieldOptions().maxAnalyzedOffset(), + maxAnalyzedOffset + ); Analyzer analyzer = wrapAnalyzer( context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER), queryMaxAnalyzedOffset @@ -119,7 +123,8 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc for (Object textToHighlight : textsToHighlight) { String text = convertFieldValue(fieldType, textToHighlight); int textLength = text.length(); - if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset) && (textLength > maxAnalyzedOffset)) { + if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset) + && (textLength > maxAnalyzedOffset)) { throw new IllegalArgumentException( "The length [" + textLength @@ -241,9 +246,9 @@ private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer an } } - private static Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) { + private static Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) { if (maxAnalyzedOffset != null) { - return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset); + return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull()); } return analyzer; } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 40da2e2a03a77..bdc75b4b0d63c 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -144,7 +144,6 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution RescorePhase.execute(searchContext); SuggestPhase.execute(searchContext); - if (searchContext.getProfilers() != null) { searchContext.queryResult().profileResults(searchContext.getProfilers().buildQueryPhaseResults()); } diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java index f8b348b383f01..fdd5efceaae3c 100644 --- a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java @@ -72,7 +72,7 @@ public static void execute(SearchContext context) { assert topDocsSortedByScore(topDocs) : "topdocs should be sorted after rescore"; ctx.setCancellationChecker(null); } - /** + /* * Since rescorers are building top docs with score only, we must reconstruct the {@link TopFieldGroups} * or {@link TopFieldDocs} using their original version before rescoring. */ @@ -86,12 +86,13 @@ public static void execute(SearchContext context) { .topDocs(new TopDocsAndMaxScore(topDocs, topDocs.scoreDocs[0].score), context.queryResult().sortValueFormats()); } catch (IOException e) { throw new ElasticsearchException("Rescore Phase Failed", e); - } catch (ContextIndexSearcher.TimeExceededException e) { + } catch (ContextIndexSearcher.TimeExceededException timeExceededException) { SearchTimeoutException.handleTimeout( context.request().allowPartialSearchResults(), context.shardTarget(), context.queryResult() ); + // if the rescore phase times out and partial results are allowed, the returned top docs from this shard won't be rescored } } diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index cd597f3328c0f..f1ca48b7d3dda 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -749,4 +749,11 @@ public FieldSortBuilder rewrite(QueryRewriteContext ctx) throws IOException { } return new FieldSortBuilder(this).setNestedSort(rewrite); } + + @Override + public boolean supportsParallelCollection() { + // Disable parallel collection for sort by field. + // It is supported but not optimized on the Lucene side to share info across collectors, and can cause regressions. + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 9bd5d4b1a23fa..ab96638efe626 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -746,4 +746,11 @@ public GeoDistanceSortBuilder rewrite(QueryRewriteContext ctx) throws IOExceptio } return new GeoDistanceSortBuilder(this).setNestedSort(rewrite); } + + @Override + public boolean supportsParallelCollection() { + // Disable parallel collection for sort by field. + // It is supported but not optimized on the Lucene side to share info across collectors, and can cause regressions. + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index a6fd4ef90693d..0c2fc68e7e856 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -173,9 +173,4 @@ public TransportVersion getMinimalSupportedVersion() { public ScoreSortBuilder rewrite(QueryRewriteContext ctx) { return this; } - - @Override - public boolean supportsParallelCollection() { - return true; - } } diff --git a/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index 48773eec8371b..c5ebf97183601 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -53,6 +54,7 @@ import java.io.IOException; import java.util.Locale; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.search.sort.FieldSortBuilder.validateMaxChildrenExistOnlyInTopLevelNestedSort; @@ -287,11 +289,13 @@ private IndexFieldData.XFieldComparatorSource fieldComparatorSource(SearchExecut final StringSortScript.Factory factory = context.compile(script, StringSortScript.CONTEXT); final StringSortScript.LeafFactory searchScript = factory.newFactory(script.getParams()); return new BytesRefFieldComparatorSource(null, null, valueMode, nested) { - StringSortScript leafScript; + final Map leafScripts = ConcurrentCollections.newConcurrentMap(); @Override protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { - leafScript = searchScript.newInstance(new DocValuesDocReader(searchLookup, context)); + // we may see the same leaf context multiple times, and each time we need to refresh the doc values doc reader + StringSortScript leafScript = searchScript.newInstance(new DocValuesDocReader(searchLookup, context)); + leafScripts.put(context.id(), leafScript); final BinaryDocValues values = new AbstractBinaryDocValues() { final BytesRefBuilder spare = new BytesRefBuilder(); @@ -311,8 +315,8 @@ public BytesRef binaryValue() { } @Override - protected void setScorer(Scorable scorer) { - leafScript.setScorer(scorer); + protected void setScorer(LeafReaderContext context, Scorable scorer) { + leafScripts.get(context.id()).setScorer(scorer); } @Override @@ -335,13 +339,15 @@ public BucketedSort newBucketedSort( case NUMBER -> { final NumberSortScript.Factory numberSortFactory = context.compile(script, NumberSortScript.CONTEXT); // searchLookup is unnecessary here, as it's just used for expressions - final NumberSortScript.LeafFactory numberSortScript = numberSortFactory.newFactory(script.getParams(), searchLookup); + final NumberSortScript.LeafFactory numberSortScriptFactory = numberSortFactory.newFactory(script.getParams(), searchLookup); return new DoubleValuesComparatorSource(null, Double.MAX_VALUE, valueMode, nested) { - NumberSortScript leafScript; + final Map leafScripts = ConcurrentCollections.newConcurrentMap(); @Override protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException { - leafScript = numberSortScript.newInstance(new DocValuesDocReader(searchLookup, context)); + // we may see the same leaf context multiple times, and each time we need to refresh the doc values doc reader + NumberSortScript leafScript = numberSortScriptFactory.newInstance(new DocValuesDocReader(searchLookup, context)); + leafScripts.put(context.id(), leafScript); final NumericDoubleValues values = new NumericDoubleValues() { @Override public boolean advanceExact(int doc) { @@ -358,8 +364,8 @@ public double doubleValue() { } @Override - protected void setScorer(Scorable scorer) { - leafScript.setScorer(scorer); + protected void setScorer(LeafReaderContext context, Scorable scorer) { + leafScripts.get(context.id()).setScorer(scorer); } }; } @@ -367,11 +373,13 @@ protected void setScorer(Scorable scorer) { final BytesRefSortScript.Factory factory = context.compile(script, BytesRefSortScript.CONTEXT); final BytesRefSortScript.LeafFactory searchScript = factory.newFactory(script.getParams()); return new BytesRefFieldComparatorSource(null, null, valueMode, nested) { - BytesRefSortScript leafScript; + final Map leafScripts = ConcurrentCollections.newConcurrentMap(); @Override protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { - leafScript = searchScript.newInstance(new DocValuesDocReader(searchLookup, context)); + // we may see the same leaf context multiple times, and each time we need to refresh the doc values doc reader + BytesRefSortScript leafScript = searchScript.newInstance(new DocValuesDocReader(searchLookup, context)); + leafScripts.put(context.id(), leafScript); final BinaryDocValues values = new AbstractBinaryDocValues() { @Override @@ -400,8 +408,8 @@ public BytesRef binaryValue() { } @Override - protected void setScorer(Scorable scorer) { - leafScript.setScorer(scorer); + protected void setScorer(LeafReaderContext context, Scorable scorer) { + leafScripts.get(context.id()).setScorer(scorer); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index 7ed3abe7daa1a..17da9abd3952b 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -290,6 +290,6 @@ public String toString() { } public boolean supportsParallelCollection() { - return false; + return true; } } diff --git a/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java index d63e0717ca7ac..17ff07f167ff8 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java @@ -10,7 +10,9 @@ import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.query.SearchTimeoutException; import org.elasticsearch.search.suggest.Suggest.Suggestion; import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry; import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option; @@ -40,12 +42,17 @@ public static void execute(SearchContext context) { for (Map.Entry entry : suggest.suggestions().entrySet()) { SuggestionSearchContext.SuggestionContext suggestion = entry.getValue(); Suggester suggester = suggestion.getSuggester(); - Suggestion> result = suggester.execute( - entry.getKey(), - suggestion, - context.searcher(), - spare - ); + Suggestion> result; + try { + result = suggester.execute(entry.getKey(), suggestion, context.searcher(), spare); + } catch (ContextIndexSearcher.TimeExceededException timeExceededException) { + SearchTimeoutException.handleTimeout( + context.request().allowPartialSearchResults(), + context.shardTarget(), + context.queryResult() + ); + result = suggester.emptySuggestion(entry.getKey(), suggestion, spare); + } if (result != null) { assert entry.getKey().equals(result.name); suggestions.add(result); @@ -56,5 +63,4 @@ public static void execute(SearchContext context) { throw new ElasticsearchException("I/O exception during suggest phase", e); } } - } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java index 2855fe8bcf0eb..184bc3024aa65 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java @@ -17,12 +17,14 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import java.io.IOException; import java.util.Arrays; +import java.util.Comparator; import java.util.Objects; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -30,9 +32,8 @@ /** * A query that matches the provided docs with their scores. * - * Note: this query was adapted from Lucene's DocAndScoreQuery from the class + * Note: this query was originally adapted from Lucene's DocAndScoreQuery from the class * {@link org.apache.lucene.search.KnnFloatVectorQuery}, which is package-private. - * There are no changes to the behavior, just some renames. */ public class KnnScoreDocQuery extends Query { private final int[] docs; @@ -49,13 +50,18 @@ public class KnnScoreDocQuery extends Query { /** * Creates a query. * - * @param docs the global doc IDs of documents that match, in ascending order - * @param scores the scores of the matching documents + * @param scoreDocs an array of ScoreDocs to use for the query * @param reader IndexReader */ - KnnScoreDocQuery(int[] docs, float[] scores, IndexReader reader) { - this.docs = docs; - this.scores = scores; + KnnScoreDocQuery(ScoreDoc[] scoreDocs, IndexReader reader) { + // Ensure that the docs are sorted by docId, as they are later searched using binary search + Arrays.sort(scoreDocs, Comparator.comparingInt(scoreDoc -> scoreDoc.doc)); + this.docs = new int[scoreDocs.length]; + this.scores = new float[scoreDocs.length]; + for (int i = 0; i < scoreDocs.length; i++) { + docs[i] = scoreDocs[i].doc; + scores[i] = scoreDocs[i].score; + } this.segmentStarts = findSegmentStarts(reader, docs); this.contextIdentity = reader.getContext().id(); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java index 6fa83ccfb6ac2..1a81f4b984e93 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java @@ -141,15 +141,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected Query doToQuery(SearchExecutionContext context) throws IOException { - int numDocs = scoreDocs.length; - int[] docs = new int[numDocs]; - float[] scores = new float[numDocs]; - for (int i = 0; i < numDocs; i++) { - docs[i] = scoreDocs[i].doc; - scores[i] = scoreDocs[i].score; - } - - return new KnnScoreDocQuery(docs, scores, context.getIndexReader()); + return new KnnScoreDocQuery(scoreDocs, context.getIndexReader()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java index e107db3a5b328..ff6e869b07131 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreKnnVectorQuery.java @@ -16,7 +16,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.elasticsearch.index.mapper.vectors.VectorSimilarityFloatValueSource; import org.elasticsearch.search.profile.query.QueryProfiler; @@ -59,15 +58,7 @@ public Query rewrite(IndexSearcher searcher) throws IOException { // Retrieve top k documents from the rescored query TopDocs topDocs = searcher.search(query, k); vectorOperations = topDocs.totalHits.value; - ScoreDoc[] scoreDocs = topDocs.scoreDocs; - int[] docIds = new int[scoreDocs.length]; - float[] scores = new float[scoreDocs.length]; - for (int i = 0; i < scoreDocs.length; i++) { - docIds[i] = scoreDocs[i].doc; - scores[i] = scoreDocs[i].score; - } - - return new KnnScoreDocQuery(docIds, scores, searcher.getIndexReader()); + return new KnnScoreDocQuery(topDocs.scoreDocs, searcher.getIndexReader()); } public Query innerQuery() { diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index c84245b34698a..33d2c74b71b76 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.RepositoryMetadata; @@ -586,7 +587,7 @@ private static Tuple, Map> getD Collection featureStateDataStreams, boolean includeAliases ) { - Map dataStreams; + Map allDataStreams; Map dataStreamAliases; List requestedDataStreams = filterIndices( snapshotInfo.dataStreams(), @@ -594,20 +595,21 @@ private static Tuple, Map> getD IndicesOptions.lenientExpand() ); if (requestedDataStreams.isEmpty()) { - dataStreams = Map.of(); + allDataStreams = Map.of(); dataStreamAliases = Map.of(); } else { if (globalMetadata == null) { globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); } final Map dataStreamsInSnapshot = globalMetadata.dataStreams(); - dataStreams = Maps.newMapWithExpectedSize(requestedDataStreams.size()); + allDataStreams = Maps.newMapWithExpectedSize(requestedDataStreams.size()); + Map systemDataStreams = new HashMap<>(); for (String requestedDataStream : requestedDataStreams) { final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream); assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot"; if (dataStreamInSnapshot.isSystem() == false) { - dataStreams.put(requestedDataStream, dataStreamInSnapshot); + allDataStreams.put(requestedDataStream, dataStreamInSnapshot); } else if (requestIndices.contains(requestedDataStream)) { throw new IllegalArgumentException( format( @@ -616,7 +618,8 @@ private static Tuple, Map> getD ) ); } else if (featureStateDataStreams.contains(requestedDataStream)) { - dataStreams.put(requestedDataStream, dataStreamInSnapshot); + allDataStreams.put(requestedDataStream, dataStreamInSnapshot); + systemDataStreams.put(requestedDataStream, dataStreamInSnapshot); } else { logger.debug( "omitting system data stream [{}] from snapshot restoration because its feature state was not requested", @@ -624,11 +627,12 @@ private static Tuple, Map> getD ); } } - if (includeAliases) { + if (includeAliases || systemDataStreams.isEmpty() == false) { dataStreamAliases = new HashMap<>(); final Map dataStreamAliasesInSnapshot = globalMetadata.dataStreamAliases(); + Map dataStreamsWithAliases = includeAliases ? allDataStreams : systemDataStreams; for (DataStreamAlias alias : dataStreamAliasesInSnapshot.values()) { - DataStreamAlias copy = alias.intersect(dataStreams.keySet()::contains); + DataStreamAlias copy = alias.intersect(dataStreamsWithAliases.keySet()::contains); if (copy.getDataStreams().isEmpty() == false) { dataStreamAliases.put(alias.getName(), copy); } @@ -637,7 +641,7 @@ private static Tuple, Map> getD dataStreamAliases = Map.of(); } } - return new Tuple<>(dataStreams, dataStreamAliases); + return new Tuple<>(allDataStreams, dataStreamAliases); } private Map> getFeatureStatesToRestore( @@ -730,6 +734,29 @@ private Set resolveSystemIndicesToDelete(ClusterState currentState, Set resolveSystemDataStreamsToDelete(ClusterState currentState, Set featureStatesToRestore) { + if (featureStatesToRestore == null) { + return Collections.emptySet(); + } + + return featureStatesToRestore.stream() + .map(systemIndices::getFeature) + .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore` + .flatMap(feature -> feature.getDataStreamDescriptors().stream()) + .map(SystemDataStreamDescriptor::getDataStreamName) + .filter(datastreamName -> currentState.metadata().dataStreams().containsKey(datastreamName)) + .map(dataStreamName -> currentState.metadata().dataStreams().get(dataStreamName)) + .collect(Collectors.toUnmodifiableSet()); + } + // visible for testing static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metadata, RestoreSnapshotRequest request) { String dataStreamName = dataStream.getName(); @@ -1301,6 +1328,13 @@ public ClusterState execute(ClusterState currentState) { settings ); + // Clear out all existing system data streams + currentState = MetadataDataStreamsService.deleteDataStreams( + currentState, + resolveSystemDataStreamsToDelete(currentState, featureStatesToRestore), + settings + ); + // List of searchable snapshots indices to restore final Set searchableSnapshotsIndices = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 008c75ed13473..22df4e2804e59 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -2517,8 +2517,6 @@ public void onFailure(Exception e) { Strings.collectionToDelimitedStringWithLimit( deleteEntry.snapshots().stream().map(SnapshotId::getName).toList(), ",", - "", - "", 1024, sb ); @@ -3885,6 +3883,11 @@ public void onFailure(Exception e) { logSnapshotFailure("create", snapshot, e); listener.onFailure(e); } + + @Override + public String toString() { + return "CreateSnapshotTask{repository=" + repository.getMetadata().name() + ", snapshot=" + snapshot + '}'; + } } private static void logSnapshotFailure(String operation, Snapshot snapshot, Exception e) { @@ -4081,7 +4084,15 @@ private SnapshotsInProgress createSnapshot( if (featureSystemIndices.size() > 0 || featureAssociatedIndices.size() > 0 || featureDataStreamBackingIndices.size() > 0) { - featureStates.add(new SnapshotFeatureInfo(featureName, List.copyOf(featureSystemIndices))); + featureStates.add( + new SnapshotFeatureInfo( + featureName, + List.copyOf( + Stream.concat(featureSystemIndices.stream(), featureDataStreamBackingIndices.stream()) + .collect(Collectors.toSet()) + ) + ) + ); indexNames.addAll(featureSystemIndices); indexNames.addAll(featureAssociatedIndices); indexNames.addAll(featureDataStreamBackingIndices); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java index 81c7bec678b3a..4a56e96aa3f30 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancelledException.java @@ -27,6 +27,11 @@ public TaskCancelledException(StreamInput in) throws IOException { super(in); } + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } + @Override public RestStatus status() { // Tasks are typically cancelled at the request of the client, so a 4xx status code is more accurate than the default of 500 (and diff --git a/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java b/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java index 1017d41a77444..0fb2f1e471d0b 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java @@ -105,9 +105,14 @@ public ScalingExecutorBuilder( final EsExecutors.TaskTrackingConfig trackingConfig ) { super(name, false); - this.coreSetting = Setting.intSetting(settingsKey(prefix, "core"), core, Setting.Property.NodeScope); - this.maxSetting = Setting.intSetting(settingsKey(prefix, "max"), max, Setting.Property.NodeScope); - this.keepAliveSetting = Setting.timeSetting(settingsKey(prefix, "keep_alive"), keepAlive, Setting.Property.NodeScope); + this.coreSetting = Setting.intSetting(settingsKey(prefix, "core"), core, 0, Setting.Property.NodeScope); + this.maxSetting = Setting.intSetting(settingsKey(prefix, "max"), max, 1, Setting.Property.NodeScope); + this.keepAliveSetting = Setting.timeSetting( + settingsKey(prefix, "keep_alive"), + keepAlive, + TimeValue.ZERO, + Setting.Property.NodeScope + ); this.rejectAfterShutdown = rejectAfterShutdown; this.trackingConfig = trackingConfig; } @@ -172,5 +177,4 @@ static class ScalingExecutorSettings extends ExecutorBuilder.ExecutorSettings { this.keepAlive = keepAlive; } } - } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 8399f5dd72f7d..95e507f70d7a9 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -157,15 +157,15 @@ protected Map> groupClusterIndices(Set remoteCluste if (indexName.equals("*") == false) { throw new IllegalArgumentException( Strings.format( - "To exclude a cluster you must specify the '*' wildcard for " + "the index expression, but found: [%s]", + "To exclude a cluster you must specify the '*' wildcard for the index expression, but found: [%s]", indexName ) ); } - if (selectorString != null && selectorString.equals("*") == false) { + if (selectorString != null) { throw new IllegalArgumentException( Strings.format( - "To exclude a cluster you must specify the '::*' selector or leave it off, but found: [%s]", + "To exclude a cluster you must not specify the a selector, but found selector: [%s]", selectorString ) ); diff --git a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java index 3683b89c922a2..baf40c1ba03fb 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java @@ -18,8 +18,14 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.metrics.CounterMetric; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; import java.io.EOFException; @@ -157,6 +163,9 @@ final class TransportHandshaker { * [3] Parent task ID should be empty; see org.elasticsearch.tasks.TaskId.writeTo for its structure. */ + private static final Logger logger = LogManager.getLogger(TransportHandshaker.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); + static final TransportVersion V7_HANDSHAKE_VERSION = TransportVersion.fromId(6_08_00_99); static final TransportVersion V8_HANDSHAKE_VERSION = TransportVersion.fromId(7_17_00_99); static final TransportVersion V9_HANDSHAKE_VERSION = TransportVersion.fromId(8_800_00_0); @@ -167,6 +176,7 @@ final class TransportHandshaker { ); static final String HANDSHAKE_ACTION_NAME = "internal:tcp/handshake"; + static final TransportVersion V8_18_FIRST_VERSION = TransportVersions.INDEXING_PRESSURE_THROTTLING_STATS; private final ConcurrentMap pendingHandshakes = new ConcurrentHashMap<>(); private final CounterMetric numHandshakes = new CounterMetric(); @@ -195,7 +205,7 @@ void sendHandshake( ActionListener listener ) { numHandshakes.inc(); - final HandshakeResponseHandler handler = new HandshakeResponseHandler(requestId, listener); + final HandshakeResponseHandler handler = new HandshakeResponseHandler(requestId, channel, listener); pendingHandshakes.put(requestId, handler); channel.addCloseListener( ActionListener.running(() -> handler.handleLocalException(new TransportException("handshake failed because connection reset"))) @@ -221,9 +231,9 @@ void sendHandshake( } void handleHandshake(TransportChannel channel, long requestId, StreamInput stream) throws IOException { + final HandshakeRequest handshakeRequest; try { - // Must read the handshake request to exhaust the stream - new HandshakeRequest(stream); + handshakeRequest = new HandshakeRequest(stream); } catch (Exception e) { assert ignoreDeserializationErrors : e; throw e; @@ -242,7 +252,101 @@ void handleHandshake(TransportChannel channel, long requestId, StreamInput strea assert ignoreDeserializationErrors : exception; throw exception; } - channel.sendResponse(new HandshakeResponse(this.version, Build.current().version())); + channel.sendResponse( + new HandshakeResponse( + ensureCompatibleVersion( + version, + handshakeRequest.transportVersion, + handshakeRequest.releaseVersion, + channel, + threadPool.getThreadContext() + ), + Build.current().version() + ) + ); + } + + private static TransportVersion ensureCompatibleVersion( + TransportVersion localTransportVersion, + TransportVersion remoteTransportVersion, + String remoteReleaseVersion, + Object channel, + ThreadContext threadContext + ) { + if (TransportVersion.isCompatible(remoteTransportVersion)) { + // Prevent log message headers from being added to the handshake response. + try (var ignored = threadContext.stashContext()) { + if (remoteTransportVersion.before(V8_18_FIRST_VERSION)) { + deprecationLogger.warn( + DeprecationCategory.OTHER, + "handshake_version", + getDeprecationMessage(localTransportVersion, remoteTransportVersion, remoteReleaseVersion, channel) + ); + } + } + if (remoteTransportVersion.onOrAfter(localTransportVersion)) { + // Remote is semantically newer than us (i.e. has a greater transport protocol version), so we propose using our current + // transport protocol version. If we're initiating the connection then that's the version we'll use; if the other end is + // initiating the connection then it's up to the other end to decide whether to use this version (if it knows it) or + // an earlier one. + return localTransportVersion; + } + final var bestKnownVersion = remoteTransportVersion.bestKnownVersion(); + if (bestKnownVersion.equals(TransportVersions.ZERO) == false) { + if (bestKnownVersion.equals(remoteTransportVersion) == false) { + // Remote is semantically older than us (i.e. has a lower transport protocol version), but we do not know its exact + // transport protocol version so it must be chronologically newer. We recommend not doing this, it implies an upgrade + // that goes backwards in time and therefore may regress in some way, so we emit a warning. But we carry on with the + // best known version anyway since both ends will know it. + logger.warn( + """ + Negotiating transport handshake with remote node with version [{}/{}] received on [{}] which appears to be \ + from a chronologically-older release with a numerically-newer version compared to this node's version [{}/{}]. \ + Upgrading to a chronologically-older release may not work reliably and is not recommended. \ + Falling back to transport protocol version [{}].""", + remoteReleaseVersion, + remoteTransportVersion, + channel, + Build.current().version(), + localTransportVersion, + bestKnownVersion + ); + } // else remote is semantically older and we _do_ know its version, so we just use that without further fuss. + return bestKnownVersion; + } + } + + final var message = Strings.format( + """ + Rejecting unreadable transport handshake from remote node with version [%s/%s] received on [%s] since this node has \ + version [%s/%s] which has an incompatible wire format.""", + remoteReleaseVersion, + remoteTransportVersion, + channel, + Build.current().version(), + localTransportVersion + ); + logger.warn(message); + throw new IllegalStateException(message); + + } + + // Non-private for testing + static String getDeprecationMessage( + TransportVersion localTransportVersion, + TransportVersion remoteTransportVersion, + String remoteReleaseVersion, + Object channel + ) { + return Strings.format( + "Performed a handshake with a remote node with version [%s/%s] received on [%s] which " + + "will be incompatible after this node on version [%s/%s] is upgraded to 9.x.", + remoteReleaseVersion, + remoteTransportVersion, + channel, + Build.current().version(), + localTransportVersion + ); } TransportResponseHandler removeHandlerForHandshake(long requestId) { @@ -260,11 +364,13 @@ long getNumHandshakes() { private class HandshakeResponseHandler implements TransportResponseHandler { private final long requestId; + private final TcpChannel channel; private final ActionListener listener; private final AtomicBoolean isDone = new AtomicBoolean(false); - private HandshakeResponseHandler(long requestId, ActionListener listener) { + private HandshakeResponseHandler(long requestId, TcpChannel channel, ActionListener listener) { this.requestId = requestId; + this.channel = channel; this.listener = listener; } @@ -281,20 +387,18 @@ public Executor executor() { @Override public void handleResponse(HandshakeResponse response) { if (isDone.compareAndSet(false, true)) { - TransportVersion responseVersion = response.transportVersion; - if (TransportVersion.isCompatible(responseVersion) == false) { - listener.onFailure( - new IllegalStateException( - "Received message from unsupported version: [" - + responseVersion - + "] minimal compatible version is: [" - + TransportVersions.MINIMUM_COMPATIBLE - + "]" - ) + ActionListener.completeWith(listener, () -> { + final var resultVersion = ensureCompatibleVersion( + version, + response.getTransportVersion(), + response.getReleaseVersion(), + channel, + threadPool.getThreadContext() ); - } else { - listener.onResponse(TransportVersion.min(TransportHandshaker.this.version, response.getTransportVersion())); - } + assert TransportVersion.current().before(version) // simulating a newer-version transport service for test purposes + || resultVersion.isKnown() : "negotiated unknown version " + resultVersion; + return resultVersion; + }); } } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index af9887d8a16c8..5d490fdc70de4 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -84,7 +84,7 @@ public class TransportService extends AbstractLifecycleComponent /** * A feature flag enabling transport upgrades for serverless. */ - private static final String SERVERLESS_TRANSPORT_SYSTEM_PROPERTY = "es.serverless_transport"; + static final String SERVERLESS_TRANSPORT_SYSTEM_PROPERTY = "es.serverless_transport"; private static final boolean SERVERLESS_TRANSPORT_FEATURE_FLAG = Booleans.parseBoolean( System.getProperty(SERVERLESS_TRANSPORT_SYSTEM_PROPERTY), false diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java deleted file mode 100644 index 4768a5cae7dad..0000000000000 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.upgrades; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.client.internal.OriginSettingClient; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.SystemIndexDescriptor; -import org.elasticsearch.indices.SystemIndices; -import org.elasticsearch.plugins.SystemIndexPlugin; - -import java.util.Comparator; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.State.CLOSE; -import static org.elasticsearch.core.Strings.format; - -/** - * Holds the data required to migrate a single system index, including metadata from the current index. If necessary, computes the settings - * and mappings for the "next" index based off of the current one. - */ -class SystemIndexMigrationInfo implements Comparable { - private static final Logger logger = LogManager.getLogger(SystemIndexMigrationInfo.class); - - private final IndexMetadata currentIndex; - private final String featureName; - private final Settings settings; - private final String mapping; - private final String origin; - private final String migrationScript; - private final SystemIndices.Feature owningFeature; - private final boolean allowsTemplates; - - private static final Comparator SAME_CLASS_COMPARATOR = Comparator.comparing( - SystemIndexMigrationInfo::getFeatureName - ).thenComparing(SystemIndexMigrationInfo::getCurrentIndexName); - - private SystemIndexMigrationInfo( - IndexMetadata currentIndex, - String featureName, - Settings settings, - String mapping, - String origin, - String migrationScript, - SystemIndices.Feature owningFeature, - boolean allowsTemplates - ) { - this.currentIndex = currentIndex; - this.featureName = featureName; - this.settings = settings; - this.mapping = mapping; - this.origin = origin; - this.migrationScript = migrationScript; - this.owningFeature = owningFeature; - this.allowsTemplates = allowsTemplates; - } - - /** - * Gets the name of the index to be migrated. - */ - String getCurrentIndexName() { - return currentIndex.getIndex().getName(); - } - - /** - * Indicates if the index to be migrated is closed. - */ - boolean isCurrentIndexClosed() { - return CLOSE.equals(currentIndex.getState()); - } - - /** - * Gets the name to be used for the post-migration index. - */ - String getNextIndexName() { - return currentIndex.getIndex().getName() + SystemIndices.UPGRADED_INDEX_SUFFIX; - } - - /** - * Gets the name of the feature which owns the index to be migrated. - */ - String getFeatureName() { - return featureName; - } - - /** - * Gets the mappings to be used for the post-migration index. - */ - String getMappings() { - return mapping; - } - - /** - * Gets the settings to be used for the post-migration index. - */ - Settings getSettings() { - return settings; - } - - /** - * Gets the origin that should be used when interacting with this index. - */ - String getOrigin() { - return origin; - } - - String getMigrationScript() { - return migrationScript; - } - - /** - * By default, system indices should not be affected by user defined templates, so this - * method should return false in almost all cases. At the moment certain Kibana indices use - * templates, therefore we allow templates to be used on Kibana created system indices until - * Kibana removes the template use on system index creation. - */ - boolean allowsTemplates() { - return allowsTemplates; - } - - /** - * Invokes the pre-migration hook for the feature that owns this index. - * See {@link SystemIndexPlugin#prepareForIndicesMigration(ClusterService, Client, ActionListener)}. - * @param clusterService For retrieving the state. - * @param client For performing any update operations necessary to prepare for the upgrade. - * @param listener Call {@link ActionListener#onResponse(Object)} when preparation for migration is complete. - */ - void prepareForIndicesMigration(ClusterService clusterService, Client client, ActionListener> listener) { - owningFeature.getPreMigrationFunction().prepareForIndicesMigration(clusterService, client, listener); - } - - /** - * Invokes the post-migration hooks for the feature that owns this index. - * See {@link SystemIndexPlugin#indicesMigrationComplete(Map, ClusterService, Client, ActionListener)}. - * @param metadata The metadata that was passed into the listener by the pre-migration hook. - * @param clusterService For retrieving the state. - * @param client For performing any update operations necessary to prepare for the upgrade. - * @param listener Call {@link ActionListener#onResponse(Object)} when the hook is finished. - */ - void indicesMigrationComplete( - Map metadata, - ClusterService clusterService, - Client client, - ActionListener listener - ) { - owningFeature.getPostMigrationFunction().indicesMigrationComplete(metadata, clusterService, client, listener); - } - - /** - * Creates a client that's been configured to be able to properly access the system index to be migrated. - * @param baseClient The base client to wrap. - * @return An {@link OriginSettingClient} which uses the origin provided by {@link SystemIndexMigrationInfo#getOrigin()}. - */ - Client createClient(Client baseClient) { - return new OriginSettingClient(baseClient, this.getOrigin()); - } - - @Override - public int compareTo(SystemIndexMigrationInfo o) { - return SAME_CLASS_COMPARATOR.compare(this, o); - } - - @Override - public String toString() { - return "IndexUpgradeInfo[" - + "currentIndex='" - + currentIndex.getIndex().getName() - + "\'" - + ", featureName='" - + featureName - + '\'' - + ", settings=" - + settings - + ", mapping='" - + mapping - + '\'' - + ", origin='" - + origin - + '\''; - } - - static SystemIndexMigrationInfo build( - IndexMetadata currentIndex, - SystemIndexDescriptor descriptor, - SystemIndices.Feature feature, - IndexScopedSettings indexScopedSettings - ) { - final Settings settings; - final String mapping; - if (descriptor.isAutomaticallyManaged()) { - Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put(descriptor.getSettings()); - settingsBuilder.remove(IndexMetadata.SETTING_VERSION_CREATED); // Simplifies testing, should never impact real uses. - settings = settingsBuilder.build(); - - mapping = descriptor.getMappings(); - } else { - // Get Settings from old index - settings = copySettingsForNewIndex(currentIndex.getSettings(), indexScopedSettings); - - // Copy mapping from the old index - mapping = currentIndex.mapping().source().string(); - } - return new SystemIndexMigrationInfo( - currentIndex, - feature.getName(), - settings, - mapping, - descriptor.getOrigin(), - descriptor.getMigrationScript(), - feature, - descriptor.allowsTemplates() - ); - } - - private static Settings copySettingsForNewIndex(Settings currentIndexSettings, IndexScopedSettings indexScopedSettings) { - Settings.Builder newIndexSettings = Settings.builder(); - currentIndexSettings.keySet() - .stream() - .filter(settingKey -> indexScopedSettings.isPrivateSetting(settingKey) == false) - .map(indexScopedSettings::get) - .filter(Objects::nonNull) - .filter(setting -> setting.getProperties().contains(Setting.Property.NotCopyableOnResize) == false) - .filter(setting -> setting.getProperties().contains(Setting.Property.PrivateIndex) == false) - .forEach(setting -> { - newIndexSettings.put(setting.getKey(), currentIndexSettings.get(setting.getKey())); - }); - return newIndexSettings.build(); - } - - /** - * Convenience factory method holding the logic for creating instances from a Feature object. - * @param feature The feature that - * @param metadata The current metadata, as index migration depends on the current state of the cluster. - * @param indexScopedSettings This is necessary to make adjustments to the indices settings for unmanaged indices. - * @return A {@link Stream} of {@link SystemIndexMigrationInfo}s that represent all the indices the given feature currently owns. - */ - static Stream fromFeature( - SystemIndices.Feature feature, - Metadata metadata, - IndexScopedSettings indexScopedSettings - ) { - return feature.getIndexDescriptors() - .stream() - .flatMap(descriptor -> descriptor.getMatchingIndices(metadata).stream().map(metadata::index).filter(imd -> { - assert imd != null : "got null IndexMetadata for index in system index descriptor [" + descriptor.getIndexPattern() + "]"; - return Objects.nonNull(imd); - }).map(imd -> SystemIndexMigrationInfo.build(imd, descriptor, feature, indexScopedSettings))); - } - - static SystemIndexMigrationInfo fromTaskState( - SystemIndexMigrationTaskState taskState, - SystemIndices systemIndices, - Metadata metadata, - IndexScopedSettings indexScopedSettings - ) { - SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(taskState.getCurrentIndex()); - SystemIndices.Feature feature = systemIndices.getFeature(taskState.getCurrentFeature()); - IndexMetadata imd = metadata.index(taskState.getCurrentIndex()); - - // It's possible for one or both of these to happen if the executing node fails during execution and: - // 1. The task gets assigned to a node with a different set of plugins installed. - // 2. The index in question is somehow deleted before we got to it. - // The first case shouldn't happen, master nodes must have all `SystemIndexPlugins` installed. - // In the second case, we should just start over. - if (descriptor == null) { - String errorMsg = format( - "couldn't find system index descriptor for index [%s] from feature [%s], which likely means this node is missing a plugin", - taskState.getCurrentIndex(), - taskState.getCurrentFeature() - ); - logger.warn(errorMsg); - assert false : errorMsg; - throw new IllegalStateException(errorMsg); - } - - if (imd == null) { - String errorMsg = format( - "couldn't find index [%s] from feature [%s] with descriptor pattern [%s]", - taskState.getCurrentIndex(), - taskState.getCurrentFeature(), - descriptor.getIndexPattern() - ); - logger.warn(errorMsg); - assert false : errorMsg; - throw new IllegalStateException(errorMsg); - } - - return build(imd, descriptor, feature, indexScopedSettings); - } -} diff --git a/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java b/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java index b9ea8504f72ab..4620e65534d3e 100644 --- a/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java +++ b/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java @@ -20,7 +20,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.security.AccessControlException; import java.util.Arrays; import java.util.stream.StreamSupport; @@ -256,7 +255,7 @@ private Observer createChild(Path file, boolean initial) throws IOException { FileObserver child = new FileObserver(file); child.init(initial); return child; - } catch (AccessControlException e) { + } catch (SecurityException e) { // don't have permissions, use a placeholder logger.debug(() -> Strings.format("Don't have permissions to watch path [%s]", file), e); return new DeniedObserver(file); diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index bc670df1f2d77..1b50c19536541 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -75,6 +75,7 @@ 7.17.25,7172599 7.17.26,7172699 7.17.27,7172799 +7.17.28,7172899 8.0.0,8000099 8.0.1,8000199 8.1.0,8010099 @@ -140,5 +141,11 @@ 8.16.1,8772004 8.16.2,8772004 8.16.3,8772004 +8.16.4,8772004 +8.16.5,8772005 +8.16.6,8772006 8.17.0,8797002 8.17.1,8797002 +8.17.2,8797002 +8.17.3,8797003 +8.17.4,8797004 diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index 77aae99907dfc..0e206a2005e74 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -23,6 +23,13 @@ grant codeBase "${codebase.mockito-core}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.lang.RuntimePermission "getClassLoader"; + // needed when calling bytebuddy + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "net.bytebuddy.createJavaDispatcher"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.utility"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.dynamic.loading"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.type"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.method"; }; grant codeBase "${codebase.byte-buddy}" { @@ -113,8 +120,9 @@ grant codeBase "${codebase.httpasyncclient}" { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; permission java.net.SocketPermission "*", "accept,connect"; }; diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index 5cba646712c0c..72ed8c6a36f96 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -75,6 +75,7 @@ 7.17.25,7172599 7.17.26,7172699 7.17.27,7172799 +7.17.28,7172899 8.0.0,8000099 8.0.1,8000199 8.1.0,8010099 @@ -140,5 +141,11 @@ 8.16.1,8518000 8.16.2,8518000 8.16.3,8518000 +8.16.4,8518000 +8.16.5,8518000 +8.16.6,8518000 8.17.0,8521000 8.17.1,8521000 +8.17.2,8521000 +8.17.3,8521000 +8.17.4,8521000 diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java index 08b12cec2e17e..f0b0fde3af20f 100644 --- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java +++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java @@ -30,6 +30,13 @@ public class TransportVersionTests extends ESTestCase { + /** + * This test is specific for 8.18, to ensure that transport versions are backported correctly. Do not forward or backport it. + */ + public void testMaximumAllowedTransportVersion() { + assertThat(TransportVersions.LATEST_DEFINED.isPatchFrom(TransportVersions.ML_INFERENCE_IBM_WATSONX_RERANK_ADDED), is(true)); + } + public void testVersionComparison() { TransportVersion V_7_2_0 = TransportVersions.V_7_2_0; TransportVersion V_8_0_0 = TransportVersions.V_8_0_0; @@ -163,15 +170,15 @@ public void testMax() { } public void testIsPatchFrom() { - TransportVersion patchVersion = TransportVersion.fromId(8_800_00_4); - assertThat(TransportVersion.fromId(8_799_00_0).isPatchFrom(patchVersion), is(false)); - assertThat(TransportVersion.fromId(8_799_00_9).isPatchFrom(patchVersion), is(false)); - assertThat(TransportVersion.fromId(8_800_00_0).isPatchFrom(patchVersion), is(false)); - assertThat(TransportVersion.fromId(8_800_00_3).isPatchFrom(patchVersion), is(false)); - assertThat(TransportVersion.fromId(8_800_00_4).isPatchFrom(patchVersion), is(true)); - assertThat(TransportVersion.fromId(8_800_00_9).isPatchFrom(patchVersion), is(true)); - assertThat(TransportVersion.fromId(8_800_01_0).isPatchFrom(patchVersion), is(false)); - assertThat(TransportVersion.fromId(8_801_00_0).isPatchFrom(patchVersion), is(false)); + TransportVersion patchVersion = TransportVersion.fromId(8_800_0_04); + assertThat(TransportVersion.fromId(8_799_0_00).isPatchFrom(patchVersion), is(false)); + assertThat(TransportVersion.fromId(8_799_0_09).isPatchFrom(patchVersion), is(false)); + assertThat(TransportVersion.fromId(8_800_0_00).isPatchFrom(patchVersion), is(false)); + assertThat(TransportVersion.fromId(8_800_0_03).isPatchFrom(patchVersion), is(false)); + assertThat(TransportVersion.fromId(8_800_0_04).isPatchFrom(patchVersion), is(true)); + assertThat(TransportVersion.fromId(8_800_0_49).isPatchFrom(patchVersion), is(true)); + assertThat(TransportVersion.fromId(8_800_1_00).isPatchFrom(patchVersion), is(false)); + assertThat(TransportVersion.fromId(8_801_0_00).isPatchFrom(patchVersion), is(false)); } public void testVersionConstantPresent() { @@ -189,6 +196,19 @@ public void testCURRENTIsLatest() { assertThat(Collections.max(TransportVersions.getAllVersions()), is(TransportVersion.current())); } + public void testPatchVersionsStillAvailable() { + for (TransportVersion tv : TransportVersionUtils.allReleasedVersions()) { + if (tv.onOrAfter(TransportVersions.V_8_9_X) && (tv.id() % 100) > 90) { + fail( + "Transport version " + + tv + + " is nearing the limit of available patch numbers." + + " Please inform the Core/Infra team that isPatchFrom may need to be modified" + ); + } + } + } + public void testToReleaseVersion() { assertThat(TransportVersion.current().toReleaseVersion(), endsWith(Version.CURRENT.toString())); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java index f3d8f8860ba83..4eed6cf0f62e2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java @@ -32,8 +32,13 @@ import java.util.EnumSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -120,4 +125,50 @@ public void testReturnsOnlyRequestedStats() throws Exception { assertNull(response.getDiskThresholdSettings()); } } + + public void testDeduplicatesStatsComputations() throws InterruptedException { + final var requestCounter = new AtomicInteger(); + final var isExecuting = new AtomicBoolean(); + when(allocationStatsService.stats()).thenAnswer(invocation -> { + try { + assertTrue(isExecuting.compareAndSet(false, true)); + assertThat(Thread.currentThread().getName(), containsString("[management]")); + return Map.of(Integer.toString(requestCounter.incrementAndGet()), NodeAllocationStatsTests.randomNodeAllocationStats()); + } finally { + Thread.yield(); + assertTrue(isExecuting.compareAndSet(true, false)); + } + }); + + final var threads = new Thread[between(1, 5)]; + final var startBarrier = new CyclicBarrier(threads.length); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + safeAwait(startBarrier); + + final var minRequestIndex = requestCounter.get(); + + final TransportGetAllocationStatsAction.Response response = safeAwait( + l -> action.masterOperation( + mock(Task.class), + new TransportGetAllocationStatsAction.Request( + TEST_REQUEST_TIMEOUT, + TaskId.EMPTY_TASK_ID, + EnumSet.of(Metric.ALLOCATIONS) + ), + ClusterState.EMPTY_STATE, + l + ) + ); + + final var requestIndex = Integer.valueOf(response.getNodeAllocationStats().keySet().iterator().next()); + assertThat(requestIndex, greaterThanOrEqualTo(minRequestIndex)); // did not get a stale result + }, "thread-" + i); + threads[i].start(); + } + + for (final var thread : threads) { + thread.join(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java deleted file mode 100644 index e1441cc4af911..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.cluster.migration; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.SystemIndexDescriptor; -import org.elasticsearch.indices.SystemIndexDescriptorUtils; -import org.elasticsearch.indices.SystemIndices; -import org.elasticsearch.test.ESTestCase; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; - -public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase { - - public static String TEST_SYSTEM_INDEX_PATTERN = ".test*"; - private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(6000099); - private static final ClusterState CLUSTER_STATE = getClusterState(); - private static final SystemIndices.Feature FEATURE = getFeature(); - - public void testGetFeatureStatus() { - GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus status = TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus( - CLUSTER_STATE, - FEATURE - ); - - assertThat(status.getUpgradeStatus(), equalTo(MIGRATION_NEEDED)); - assertThat(status.getFeatureName(), equalTo("test-feature")); - assertThat(status.getMinimumIndexVersion(), equalTo(TEST_OLD_VERSION)); - assertThat(status.getIndexVersions(), hasSize(2)); // additional testing below - } - - public void testGetIndexInfos() { - List versions = TransportGetFeatureUpgradeStatusAction.getIndexInfos( - CLUSTER_STATE, - FEATURE - ); - - assertThat(versions, hasSize(2)); - - { - GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(0); - assertThat(version.getVersion(), equalTo(IndexVersion.current())); - assertThat(version.getIndexName(), equalTo(".test-index-1")); - } - { - GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(1); - assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); - assertThat(version.getIndexName(), equalTo(".test-index-2")); - } - } - - private static SystemIndices.Feature getFeature() { - SystemIndexDescriptor descriptor = SystemIndexDescriptorUtils.createUnmanaged(TEST_SYSTEM_INDEX_PATTERN, "descriptor for tests"); - - List descriptors = new ArrayList<>(); - descriptors.add(descriptor); - - // system indices feature object - SystemIndices.Feature feature = new SystemIndices.Feature("test-feature", "feature for tests", descriptors); - return feature; - } - - private static ClusterState getClusterState() { - IndexMetadata indexMetadata1 = IndexMetadata.builder(".test-index-1") - .settings(Settings.builder().put("index.version.created", IndexVersion.current()).build()) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - @UpdateForV9 // Once we start testing 9.x, we should update this test to use a 7.x "version created" - IndexMetadata indexMetadata2 = IndexMetadata.builder(".test-index-2") - .settings(Settings.builder().put("index.version.created", TEST_OLD_VERSION).build()) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - ClusterState clusterState = new ClusterState.Builder(ClusterState.EMPTY_STATE).metadata( - new Metadata.Builder().indices(Map.of(".test-index-1", indexMetadata1, ".test-index-2", indexMetadata2)).build() - ).build(); - return clusterState; - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 7e590dc2cdeec..d6557099190f5 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -46,6 +46,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -227,4 +228,34 @@ public void testErrorWhenCreatingNonPrimarySystemIndex() { ); } + public void testCreatingSystemIndexForMigration() { + CreateIndexRequest request = new CreateIndexRequest(); + String path = "/test"; // just to test that we pass settings + Settings settings = Settings.builder().put(SETTING_INDEX_HIDDEN, true).put(IndexMetadata.SETTING_DATA_PATH, path).build(); + request.index(MANAGED_SYSTEM_INDEX_NAME + SystemIndices.UPGRADED_INDEX_SUFFIX) + .cause(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE) + .settings(settings); + + @SuppressWarnings("unchecked") + ActionListener mockListener = mock(ActionListener.class); + + action.masterOperation(mock(Task.class), request, CLUSTER_STATE, mockListener); + + ArgumentCaptor createRequestArgumentCaptor = ArgumentCaptor.forClass( + CreateIndexClusterStateUpdateRequest.class + ); + verify(mockListener, times(0)).onFailure(any()); + verify(metadataCreateIndexService, times(1)).createIndex( + any(TimeValue.class), + any(TimeValue.class), + any(TimeValue.class), + createRequestArgumentCaptor.capture(), + any() + ); + + CreateIndexClusterStateUpdateRequest processedRequest = createRequestArgumentCaptor.getValue(); + assertTrue(processedRequest.settings().getAsBoolean(SETTING_INDEX_HIDDEN, false)); + assertThat(processedRequest.settings().get(IndexMetadata.SETTING_DATA_PATH, ""), is(path)); + } + } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index 65464c7f14a5c..6e86e896061f8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -339,23 +339,27 @@ public PostingsFormat getPostingsFormatForField(String field) { }); try (Directory dir = createNewDirectory()) { + float docsWithSuggest1FieldRatio; try (IndexWriter writer = new IndexWriter(dir, config)) { int numDocs = randomIntBetween(100, 1000); + int numDocsWithSuggest1Field = 0; for (int i = 0; i < numDocs; i++) { final Document doc = new Document(); if (randomDouble() < 0.5) { + numDocsWithSuggest1Field++; doc.add(new SuggestField("suggest_1", randomAlphaOfLength(10), randomIntBetween(1, 20))); } doc.add(new SuggestField("suggest_2", randomAlphaOfLength(10), randomIntBetween(1, 20))); writer.addDocument(doc); } + docsWithSuggest1FieldRatio = (float) numDocsWithSuggest1Field / (numDocs + numDocsWithSuggest1Field); } final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {}); assertFieldStats( "suggest_1", "inverted_index", stats.getFields().get("suggest_1").getInvertedIndexBytes(), - stats.total().totalBytes() / 3, + (long) (stats.total().totalBytes() * docsWithSuggest1FieldRatio), 0.05, 2048 ); @@ -364,7 +368,7 @@ public PostingsFormat getPostingsFormatForField(String field) { "suggest_2", "inverted_index", stats.getFields().get("suggest_2").getInvertedIndexBytes(), - stats.total().totalBytes() * 2 / 3, + (long) (stats.total().totalBytes() * (1 - docsWithSuggest1FieldRatio)), 0.05, 2048 ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 57750bb02bb14..729b1734fea22 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.indices.rollover; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -203,23 +204,23 @@ public void testAliasValidation() { Metadata metadata = metadataBuilder.build(); CreateIndexRequest req = new CreateIndexRequest(); - IllegalArgumentException exception = expectThrows( + Exception exception = expectThrows( IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, aliasWithNoWriteIndex, randomAlphaOfLength(5), req, false) + () -> MetadataRolloverService.validate(metadata, aliasWithNoWriteIndex, randomAlphaOfLength(5), req) ); assertThat(exception.getMessage(), equalTo("rollover target [" + aliasWithNoWriteIndex + "] does not point to a write index")); exception = expectThrows( IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, randomFrom(index1, index2), randomAlphaOfLength(5), req, false) + () -> MetadataRolloverService.validate(metadata, randomFrom(index1, index2), randomAlphaOfLength(5), req) ); assertThat(exception.getMessage(), equalTo("rollover target is a [concrete index] but one of [alias,data_stream] was expected")); final String aliasName = randomAlphaOfLength(5); exception = expectThrows( - IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, aliasName, randomAlphaOfLength(5), req, false) + ResourceNotFoundException.class, + () -> MetadataRolloverService.validate(metadata, aliasName, randomAlphaOfLength(5), req) ); assertThat(exception.getMessage(), equalTo("rollover target [" + aliasName + "] does not exist")); - MetadataRolloverService.validate(metadata, aliasWithWriteIndex, randomAlphaOfLength(5), req, false); + MetadataRolloverService.validate(metadata, aliasWithWriteIndex, randomAlphaOfLength(5), req); } public void testDataStreamValidation() throws IOException { @@ -232,18 +233,18 @@ public void testDataStreamValidation() throws IOException { Metadata metadata = md.build(); CreateIndexRequest req = new CreateIndexRequest(); - MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, req, false); + MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, req); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), randomAlphaOfLength(5), req, false) + () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), randomAlphaOfLength(5), req) ); assertThat(exception.getMessage(), equalTo("new index name may not be specified when rolling over a data stream")); CreateIndexRequest aliasReq = new CreateIndexRequest().alias(new Alias("no_aliases_permitted")); exception = expectThrows( IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, aliasReq, false) + () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, aliasReq) ); assertThat( exception.getMessage(), @@ -254,7 +255,7 @@ public void testDataStreamValidation() throws IOException { CreateIndexRequest mappingReq = new CreateIndexRequest().mapping(mapping); exception = expectThrows( IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, mappingReq, false) + () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, mappingReq) ); assertThat( exception.getMessage(), @@ -264,7 +265,7 @@ public void testDataStreamValidation() throws IOException { CreateIndexRequest settingReq = new CreateIndexRequest().settings(Settings.builder().put("foo", "bar")); exception = expectThrows( IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, settingReq, false) + () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, settingReq) ); assertThat( exception.getMessage(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index bc86715527132..a5f5b878cdb95 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -256,7 +256,7 @@ public void testValidation() { assertNotNull(validationException); assertEquals(1, validationException.validationErrors().size()); assertEquals( - "rollover cannot be applied to both regular and failure indices at the same time", + "Invalid index name [alias-index::*], invalid usage of :: separator, [*] is not a recognized selector", validationException.validationErrors().get(0) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index f04180bde30f2..1562833077f5b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; @@ -19,11 +20,14 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -44,6 +48,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -461,19 +466,39 @@ public void testLazyRollover() throws Exception { mockMetadataDataStreamService, dataStreamAutoShardingService ); - final PlainActionFuture future = new PlainActionFuture<>(); - RolloverRequest rolloverRequest = new RolloverRequest("logs-ds", null); - rolloverRequest.lazy(true); - transportRolloverAction.masterOperation(mock(CancellableTask.class), rolloverRequest, stateBefore, future); - RolloverResponse rolloverResponse = future.actionGet(); - assertThat(rolloverResponse.getOldIndex(), equalTo(".ds-logs-ds-000001")); - assertThat(rolloverResponse.getNewIndex(), Matchers.startsWith(".ds-logs-ds-")); - assertThat(rolloverResponse.getNewIndex(), Matchers.endsWith("-000002")); - assertThat(rolloverResponse.isLazy(), equalTo(true)); - assertThat(rolloverResponse.isDryRun(), equalTo(false)); - assertThat(rolloverResponse.isRolledOver(), equalTo(false)); - assertThat(rolloverResponse.getConditionStatus().size(), equalTo(0)); - assertThat(rolloverResponse.isAcknowledged(), is(true)); + { + // Regular lazy rollover + final PlainActionFuture future = new PlainActionFuture<>(); + RolloverRequest rolloverRequest = new RolloverRequest("logs-ds", null); + rolloverRequest.lazy(true); + transportRolloverAction.masterOperation(mock(CancellableTask.class), rolloverRequest, stateBefore, future); + RolloverResponse rolloverResponse = future.actionGet(); + assertThat(rolloverResponse.getOldIndex(), equalTo(".ds-logs-ds-000001")); + assertThat(rolloverResponse.getNewIndex(), Matchers.startsWith(".ds-logs-ds-")); + assertThat(rolloverResponse.getNewIndex(), Matchers.endsWith("-000002")); + assertThat(rolloverResponse.isLazy(), equalTo(true)); + assertThat(rolloverResponse.isDryRun(), equalTo(false)); + assertThat(rolloverResponse.isRolledOver(), equalTo(false)); + assertThat(rolloverResponse.getConditionStatus().size(), equalTo(0)); + assertThat(rolloverResponse.isAcknowledged(), is(true)); + } + { + // Dry-run lazy rollover + final PlainActionFuture future = new PlainActionFuture<>(); + RolloverRequest rolloverRequest = new RolloverRequest("logs-ds", null); + rolloverRequest.lazy(true); + rolloverRequest.dryRun(true); + transportRolloverAction.masterOperation(mock(CancellableTask.class), rolloverRequest, stateBefore, future); + RolloverResponse rolloverResponse = future.actionGet(); + assertThat(rolloverResponse.getOldIndex(), equalTo(".ds-logs-ds-000001")); + assertThat(rolloverResponse.getNewIndex(), Matchers.startsWith(".ds-logs-ds-")); + assertThat(rolloverResponse.getNewIndex(), Matchers.endsWith("-000002")); + assertThat(rolloverResponse.isLazy(), equalTo(true)); + assertThat(rolloverResponse.isDryRun(), equalTo(true)); + assertThat(rolloverResponse.isRolledOver(), equalTo(false)); + assertThat(rolloverResponse.getConditionStatus().size(), equalTo(0)); + assertThat(rolloverResponse.isAcknowledged(), is(false)); + } } public void testLazyRolloverFails() throws Exception { @@ -580,6 +605,223 @@ public void testRolloverAliasToDataStreamFails() throws Exception { assertThat(illegalStateException.getMessage(), containsString("Aliases to data streams cannot be rolled over.")); } + public void testCheckBlockForIndices() { + final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( + mock(TransportService.class), + mockClusterService, + mockThreadPool, + mockActionFilters, + mockIndexNameExpressionResolver, + rolloverService, + mockClient, + mockAllocationService, + mockMetadataDataStreamService, + dataStreamAutoShardingService + ); + final IndexMetadata.Builder indexMetadata1 = IndexMetadata.builder("my-index-1") + .putAlias(AliasMetadata.builder("my-alias").writeIndex(true).build()) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1); + final IndexMetadata indexMetadata2 = IndexMetadata.builder("my-index-2") + .settings(settings(IndexVersion.current()).put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + final ClusterState stateBefore = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().put(indexMetadata1).put(indexMetadata2, false)) + .blocks(ClusterBlocks.builder().addBlocks(indexMetadata2)) + .build(); + { + RolloverRequest rolloverRequest = new RolloverRequest("my-alias", "my-new-index"); + when(mockIndexNameExpressionResolver.concreteIndexNames(any(), any(), (IndicesRequest) any())).thenReturn( + new String[] { "my-index-1" } + ); + assertNull(transportRolloverAction.checkBlock(rolloverRequest, stateBefore)); + } + { + RolloverRequest rolloverRequest = new RolloverRequest("my-index-2", "my-new-index"); + when(mockIndexNameExpressionResolver.concreteIndexNames(any(), any(), (IndicesRequest) any())).thenReturn( + new String[] { "my-index-2" } + ); + assertNotNull(transportRolloverAction.checkBlock(rolloverRequest, stateBefore)); + } + } + + public void testCheckBlockForDataStreams() { + final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( + mock(TransportService.class), + mockClusterService, + mockThreadPool, + mockActionFilters, + mockIndexNameExpressionResolver, + rolloverService, + mockClient, + mockAllocationService, + mockMetadataDataStreamService, + dataStreamAutoShardingService + ); + String dataStreamName = randomAlphaOfLength(20); + { + // First, make sure checkBlock returns null when there are no blocks + final ClusterState clusterState = createDataStream( + dataStreamName, + false, + false, + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName, null); + assertNull(transportRolloverAction.checkBlock(rolloverRequest, clusterState)); + } + { + // Make sure checkBlock returns null when indices other than the write index have blocks + final ClusterState clusterState = createDataStream( + dataStreamName, + false, + true, + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName, null); + assertNull(transportRolloverAction.checkBlock(rolloverRequest, clusterState)); + } + { + // Make sure checkBlock returns null when indices other than the write index have blocks and we use "::data" + final ClusterState clusterState = createDataStream( + dataStreamName, + false, + true, + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName + "::data", null); + assertNull(transportRolloverAction.checkBlock(rolloverRequest, clusterState)); + } + { + // Make sure checkBlock returns an exception when the write index has a block + ClusterState clusterState = createDataStream( + dataStreamName, + true, + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName, null); + if (randomBoolean()) { + rolloverRequest.setIndicesOptions(IndicesOptions.lenientExpandOpenNoSelectors()); + } + ClusterBlockException e = transportRolloverAction.checkBlock(rolloverRequest, clusterState); + assertNotNull(e); + } + { + // Make sure checkBlock returns an exception when the write index has a block and we use "::data" + ClusterState clusterState = createDataStream( + dataStreamName, + true, + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName + "::data", null); + ClusterBlockException e = transportRolloverAction.checkBlock(rolloverRequest, clusterState); + assertNotNull(e); + } + } + + public void testCheckBlockForDataStreamFailureStores() { + final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( + mock(TransportService.class), + mockClusterService, + mockThreadPool, + mockActionFilters, + mockIndexNameExpressionResolver, + rolloverService, + mockClient, + mockAllocationService, + mockMetadataDataStreamService, + dataStreamAutoShardingService + ); + String dataStreamName = randomAlphaOfLength(20); + { + // Make sure checkBlock returns no exception when there is no failure store block + ClusterState clusterState = createDataStream(dataStreamName, randomBoolean(), randomBoolean(), true, false, false); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName + "::failures", null); + assertNull(transportRolloverAction.checkBlock(rolloverRequest, clusterState)); + } + { + // Make sure checkBlock returns an exception when the failure store write index has a block + ClusterState clusterState = createDataStream(dataStreamName, randomBoolean(), randomBoolean(), true, true, randomBoolean()); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName + "::failures", null); + assertNotNull(transportRolloverAction.checkBlock(rolloverRequest, clusterState)); + } + { + // Make sure checkBlock returns no exception when failure store non-write indices have a block + ClusterState clusterState = createDataStream(dataStreamName, randomBoolean(), randomBoolean(), true, false, true); + RolloverRequest rolloverRequest = new RolloverRequest(dataStreamName + "::failures", null); + assertNull(transportRolloverAction.checkBlock(rolloverRequest, clusterState)); + } + } + + private ClusterState createDataStream( + String dataStreamName, + boolean blockOnWriteIndex, + boolean blocksOnNonWriteIndices, + boolean includeFailureStore, + boolean blockOnFailureStoreWriteIndex, + boolean blockOnFailureStoreNonWriteIndices + ) { + ClusterState.Builder clusterStateBuilder = ClusterState.builder(ClusterName.DEFAULT); + Metadata.Builder metadataBuilder = Metadata.builder(); + ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder(); + List indices = new ArrayList<>(); + int totalIndices = randomIntBetween(1, 20); + for (int i = 0; i < totalIndices; i++) { + Settings.Builder settingsBuilder = settings(IndexVersion.current()); + if ((blockOnWriteIndex && i == totalIndices - 1) || (blocksOnNonWriteIndices && i != totalIndices - 1)) { + settingsBuilder.put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true); + } + final IndexMetadata backingIndexMetadata = IndexMetadata.builder(".ds-logs-ds-00000" + (i + 1)) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + metadataBuilder.put(backingIndexMetadata, false); + indices.add(backingIndexMetadata.getIndex()); + clusterBlocksBuilder.addBlocks(backingIndexMetadata); + } + + DataStream.Builder dataStreamBuilder = DataStream.builder(dataStreamName, indices) + .setMetadata(Map.of()) + .setIndexMode(randomFrom(IndexMode.values())); + if (includeFailureStore) { + List failureStoreIndices = new ArrayList<>(); + int totalFailureStoreIndices = randomIntBetween(1, 20); + for (int i = 0; i < totalFailureStoreIndices; i++) { + Settings.Builder settingsBuilder = settings(IndexVersion.current()); + if ((blockOnFailureStoreWriteIndex && i == totalFailureStoreIndices - 1) + || (blockOnFailureStoreNonWriteIndices && i != totalFailureStoreIndices - 1)) { + settingsBuilder.put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true); + } + final IndexMetadata failureStoreIndexMetadata = IndexMetadata.builder( + DataStream.getDefaultFailureStoreName(dataStreamName, i + 1, randomMillisUpToYear9999()) + ).settings(settingsBuilder).numberOfShards(1).numberOfReplicas(1).build(); + failureStoreIndices.add(failureStoreIndexMetadata.getIndex()); + clusterBlocksBuilder.addBlocks(failureStoreIndexMetadata); + } + dataStreamBuilder.setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStoreIndices).build()); + } + clusterStateBuilder.blocks(clusterBlocksBuilder); + final DataStream dataStream = dataStreamBuilder.build(); + metadataBuilder.put(dataStream); + return clusterStateBuilder.metadata(metadataBuilder).build(); + } + private IndicesStatsResponse createIndicesStatResponse(String indexName, long totalDocs, long primariesDocs) { final CommonStats primaryStats = mock(CommonStats.class); when(primaryStats.getDocs()).thenReturn(new DocsStats(primariesDocs, 0, between(1, 10000))); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java index fc8b52dc1bdc3..e8b7d7b9fc7f9 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -273,18 +273,22 @@ public void testMinimumVersionSameAsNewVersion() { } public void testMinimumVersionBetweenNewAndOldVersion() { - var oldVersion = new VersionInformation( - VersionUtils.getFirstVersion(), - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersionUtils.randomCompatibleVersion(random()) - ); - var newVersion = new VersionInformation( VersionUtils.maxCompatibleVersion(VersionUtils.getFirstVersion()), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current() ); + var oldVersion = new VersionInformation( + VersionUtils.randomVersionBetween( + random(), + Version.CURRENT.minimumCompatibilityVersion(), + VersionUtils.getPreviousVersion(newVersion.nodeVersion()) + ), + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersionUtils.randomCompatibleVersion(random()) + ); + var minVersion = VersionUtils.randomVersionBetween( random(), allVersions().get(allVersions().indexOf(oldVersion.nodeVersion()) + 1), diff --git a/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java b/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java index 73d4ab59ce479..585d660917e4b 100644 --- a/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/IndexComponentSelectorTests.java @@ -20,7 +20,7 @@ public class IndexComponentSelectorTests extends ESTestCase { public void testIndexComponentSelectorFromKey() { assertThat(IndexComponentSelector.getByKey("data"), equalTo(IndexComponentSelector.DATA)); assertThat(IndexComponentSelector.getByKey("failures"), equalTo(IndexComponentSelector.FAILURES)); - assertThat(IndexComponentSelector.getByKey("*"), equalTo(IndexComponentSelector.ALL_APPLICABLE)); + assertThat(IndexComponentSelector.getByKey("*"), nullValue()); assertThat(IndexComponentSelector.getByKey("d*ta"), nullValue()); assertThat(IndexComponentSelector.getByKey("_all"), nullValue()); assertThat(IndexComponentSelector.getByKey("**"), nullValue()); @@ -30,11 +30,10 @@ public void testIndexComponentSelectorFromKey() { public void testIndexComponentSelectorFromId() { assertThat(IndexComponentSelector.getById((byte) 0), equalTo(IndexComponentSelector.DATA)); assertThat(IndexComponentSelector.getById((byte) 1), equalTo(IndexComponentSelector.FAILURES)); - assertThat(IndexComponentSelector.getById((byte) 2), equalTo(IndexComponentSelector.ALL_APPLICABLE)); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> IndexComponentSelector.getById((byte) 3)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> IndexComponentSelector.getById((byte) 2)); assertThat( exception.getMessage(), - containsString("Unknown id of index component selector [3], available options are: {0=DATA, 1=FAILURES, 2=ALL_APPLICABLE}") + containsString("Unknown id of index component selector [2], available options are: {0=DATA, 1=FAILURES}") ); } diff --git a/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java b/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java index 5363722f2f49f..abbbd53dec570 100644 --- a/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java @@ -100,7 +100,7 @@ public void testAsyncAcquire() throws InterruptedException { final var completionLatch = new CountDownLatch(1); final var executorService = EsExecutors.newScaling( "test", - 0, + 1, between(1, 10), 10, TimeUnit.SECONDS, diff --git a/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchTests.java b/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchTests.java index d84ee0267251a..8eaff521068e8 100644 --- a/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchTests.java +++ b/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchTests.java @@ -9,11 +9,11 @@ package org.elasticsearch.bootstrap; -import org.elasticsearch.entitlement.runtime.policy.InboundNetworkEntitlement; -import org.elasticsearch.entitlement.runtime.policy.LoadNativeLibrariesEntitlement; -import org.elasticsearch.entitlement.runtime.policy.OutboundNetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.entitlement.runtime.policy.Scope; +import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement; import org.elasticsearch.test.ESTestCase; import java.util.List; diff --git a/server/src/test/java/org/elasticsearch/bootstrap/NoSecurityManagerTests.java b/server/src/test/java/org/elasticsearch/bootstrap/NoSecurityManagerTests.java index c1bb941020575..d7628be0d7f00 100644 --- a/server/src/test/java/org/elasticsearch/bootstrap/NoSecurityManagerTests.java +++ b/server/src/test/java/org/elasticsearch/bootstrap/NoSecurityManagerTests.java @@ -12,6 +12,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.lucene.tests.util.LuceneTestCase; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.test.GraalVMThreadsFilter; import static org.hamcrest.Matchers.is; @@ -20,6 +21,7 @@ public class NoSecurityManagerTests extends LuceneTestCase { public void testPrepopulateSecurityCaller() { + assumeTrue("security manager must be available", RuntimeVersionFeature.isSecurityManagerAvailable()); assumeTrue("Unexpected security manager:" + System.getSecurityManager(), System.getSecurityManager() == null); boolean isAtLeastJava17 = Runtime.version().feature() >= 17; boolean isPrepopulated = Security.prepopulateSecurityCaller(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java index 5ab5ed1c23e4f..0f346ff0b2326 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java @@ -13,15 +13,25 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.InvalidIndexNameException; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; +import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.either; @@ -72,11 +82,8 @@ public void testResolveIndexAbstractions() { expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("index1::data"))); // Selectors allowed, valid selector given, data selector stripped off in result since it is the default assertThat(resolveAbstractionsSelectorAllowed(List.of("index1::data")), contains("index1")); - // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default - // ** only returns ::data since expression is an index - assertThat(resolveAbstractionsSelectorAllowed(List.of("index1::*")), contains("index1")); // Selectors allowed, invalid selector given - expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index1::custom"))); + expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index1::*"))); // == Single Date Math Expressions == @@ -116,7 +123,7 @@ public void testResolveIndexAbstractions() { assertThat(resolveAbstractionsSelectorAllowed(List.of("index*::data")), containsInAnyOrder("index1", "index2")); // Selectors allowed, wildcard selector provided, data selector stripped off in result since it is the default // ** only returns ::data since expression is an index - assertThat(resolveAbstractionsSelectorAllowed(List.of("index*::*")), containsInAnyOrder("index1", "index2")); + assertThat(resolveAbstractionsSelectorAllowed(List.of("index*")), containsInAnyOrder("index1", "index2")); // Selectors allowed, invalid selector given expectThrows(InvalidIndexNameException.class, () -> resolveAbstractionsSelectorAllowed(List.of("index*::custom"))); @@ -128,11 +135,9 @@ public void testResolveIndexAbstractions() { expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("data-stream1::data"))); // Selectors allowed, valid selector given assertThat(resolveAbstractionsSelectorAllowed(List.of("data-stream1::failures")), contains("data-stream1::failures")); - // Selectors allowed, wildcard selector provided - // ** returns both ::data and ::failures since expression is a data stream - // ** data selector stripped off in result since it is the default + // Selectors allowed, data selector is not added in result since it is the default assertThat( - resolveAbstractionsSelectorAllowed(List.of("data-stream1::*")), + resolveAbstractionsSelectorAllowed(List.of("data-stream1", "data-stream1::failures")), containsInAnyOrder("data-stream1", "data-stream1::failures") ); // Selectors allowed, invalid selector given @@ -146,10 +151,9 @@ public void testResolveIndexAbstractions() { expectThrows(IllegalArgumentException.class, () -> resolveAbstractionsSelectorNotAllowed(List.of("data-stream*::data"))); // Selectors allowed, valid selector given assertThat(resolveAbstractionsSelectorAllowed(List.of("data-stream*::failures")), contains("data-stream1::failures")); - // Selectors allowed, wildcard selector provided - // ** returns both ::data and ::failures since expression is a data stream + // Selectors allowed, both ::data and ::failures are returned assertThat( - resolveAbstractionsSelectorAllowed(List.of("data-stream*::*")), + resolveAbstractionsSelectorAllowed(List.of("data-stream*", "data-stream*::failures")), containsInAnyOrder("data-stream1", "data-stream1::failures") ); // Selectors allowed, invalid selector given @@ -170,7 +174,7 @@ public void testResolveIndexAbstractions() { // Selectors allowed, wildcard selector provided // ** returns both ::data and ::failures for applicable abstractions assertThat( - resolveAbstractionsSelectorAllowed(List.of("*::*")), + resolveAbstractionsSelectorAllowed(List.of("*", "*::failures")), containsInAnyOrder("index1", "index2", "data-stream1", "data-stream1::failures") ); // Selectors allowed, invalid selector given @@ -185,11 +189,11 @@ public void testResolveIndexAbstractions() { // Selectors allowed, wildcard selector provided // ** returns both ::data and ::failures for applicable abstractions // ** limits the returned values based on selectors - assertThat(resolveAbstractionsSelectorAllowed(List.of("*::*", "-*::data")), contains("data-stream1::failures")); + assertThat(resolveAbstractionsSelectorAllowed(List.of("*", "*::failures", "-*::data")), contains("data-stream1::failures")); // Selectors allowed, wildcard selector provided // ** limits the returned values based on selectors assertThat( - resolveAbstractionsSelectorAllowed(List.of("*::*", "-*::failures")), + resolveAbstractionsSelectorAllowed(List.of("*", "*::failures", "-*::failures")), containsInAnyOrder("index1", "index2", "data-stream1") ); // Selectors allowed, none given, default to both selectors @@ -215,16 +219,126 @@ public void testIsIndexVisible() { assertThat(isIndexVisible("data-stream1", "failures"), is(true)); } - private boolean isIndexVisible(String index, String selector) { - return IndexAbstractionResolver.isIndexVisible( - "*", - selector, - index, - IndicesOptions.strictExpandOpen(), - metadata, - indexNameExpressionResolver, - true + public void testIsNetNewSystemIndexVisible() { + final Settings settings = Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .build(); + + final Settings hiddenSettings = Settings.builder().put(settings).put("index.hidden", true).build(); + + final IndexMetadata foo = IndexMetadata.builder(".foo").settings(hiddenSettings).system(true).build(); + final IndexMetadata barReindexed = IndexMetadata.builder(".bar-reindexed") + .settings(hiddenSettings) + .system(true) + .putAlias(AliasMetadata.builder(".bar").isHidden(true).build()) + .build(); + final IndexMetadata other = IndexMetadata.builder("other").settings(settings).build(); + + final SystemIndexDescriptor fooDescriptor = SystemIndexDescriptor.builder() + .setDescription("foo indices") + .setOrigin("foo origin") + .setVersionMetaKey("version") + .setPrimaryIndex(".foo") + .setIndexPattern(".foo*") + .setSettings(settings) + .setMappings(mappings()) + .setNetNew() + .build(); + final SystemIndexDescriptor barDescriptor = SystemIndexDescriptor.builder() + .setDescription("bar indices") + .setOrigin("bar origin") + .setVersionMetaKey("version") + .setPrimaryIndex(".bar") + .setIndexPattern(".bar*") + .setSettings(settings) + .setMappings(mappings()) + .setNetNew() + .build(); + final SystemIndices systemIndices = new SystemIndices( + List.of(new SystemIndices.Feature("name", "description", List.of(fooDescriptor, barDescriptor))) ); + + metadata = Metadata.builder().put(foo, true).put(barReindexed, true).put(other, true).build(); + + // these indices options are for the GET _data_streams case + final IndicesOptions noHiddenNoAliases = IndicesOptions.builder() + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .build() + ) + .build(); + + { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "true"); + indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); + indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + + // this covers the GET * case -- with system access, you can see everything + assertThat(isIndexVisible("other", "*"), is(true)); + assertThat(isIndexVisible(".foo", "*"), is(true)); + assertThat(isIndexVisible(".bar", "*"), is(true)); + + // but if you don't ask for hidden and aliases, you won't see hidden indices or aliases, naturally + assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true)); + assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false)); + assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false)); + } + + { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "false"); + indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); + indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + + // this covers the GET * case -- without system access, you can't see everything + assertThat(isIndexVisible("other", "*"), is(true)); + assertThat(isIndexVisible(".foo", "*"), is(false)); + assertThat(isIndexVisible(".bar", "*"), is(false)); + + // no difference here in the datastream case, you can't see these then, either + assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true)); + assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false)); + assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false)); + } + + { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "true"); + threadContext.putHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "some-elastic-product"); + indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); + indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + + // this covers the GET * case -- with product (only) access, you can't see everything + assertThat(isIndexVisible("other", "*"), is(true)); + assertThat(isIndexVisible(".foo", "*"), is(false)); + assertThat(isIndexVisible(".bar", "*"), is(false)); + + // no difference here in the datastream case, you can't see these then, either + assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true)); + assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false)); + assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false)); + } + } + + private static XContentBuilder mappings() { + try (XContentBuilder builder = jsonBuilder()) { + return builder.startObject() + .startObject(SINGLE_MAPPING_NAME) + .startObject("_meta") + .field(SystemIndexDescriptor.VERSION_META_KEY, 0) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } private List resolveAbstractionsSelectorNotAllowed(List expressions) { @@ -238,4 +352,12 @@ private List resolveAbstractionsSelectorAllowed(List expressions private List resolveAbstractions(List expressions, IndicesOptions indicesOptions, Supplier> mask) { return indexAbstractionResolver.resolveIndexAbstractions(expressions, indicesOptions, metadata, mask, (idx) -> true, true); } + + private boolean isIndexVisible(String index, String selector) { + return isIndexVisible(index, selector, IndicesOptions.strictExpandHidden()); + } + + private boolean isIndexVisible(String index, String selector, IndicesOptions indicesOptions) { + return IndexAbstractionResolver.isIndexVisible("*", selector, index, indicesOptions, metadata, indexNameExpressionResolver, true); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index f5e4c3d8f2d09..293bdb2c53899 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -2765,10 +2765,27 @@ public void testDataStreamsWithFailureStore() { assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis))); } + // Test default with an exact data stream name and include failures true + { + IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_NO_SELECTORS; + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream"); + assertThat(result.length, equalTo(4)); + assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis))); + assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis))); + assertThat(result[2].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis))); + assertThat(result[3].getName(), equalTo(DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis))); + } + // Test explicit include failure store with an exact data stream name { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; - Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::*"); + Index[] result = indexNameExpressionResolver.concreteIndices( + state, + indicesOptions, + true, + "my-data-stream::data", + "my-data-stream::failures" + ); assertThat(result.length, equalTo(4)); assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis))); assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis))); @@ -2784,7 +2801,7 @@ public void testDataStreamsWithFailureStore() { .build(); expectThrows( IllegalArgumentException.class, - () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::*") + () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream::failures") ); } @@ -2813,6 +2830,26 @@ public void testDataStreamsWithFailureStore() { ); } + // Test default without any expressions and include failures + { + IndicesOptions indicesOptions = IndicesOptions.builder() + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowSelectors(false).includeFailureIndices(true).build()) + .build(); + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true); + assertThat(result.length, equalTo(5)); + List indexNames = Arrays.stream(result).map(Index::getName).toList(); + assertThat( + indexNames, + containsInAnyOrder( + DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis), + DataStream.getDefaultBackingIndexName(dataStreamName, 1, epochMillis), + DataStream.getDefaultFailureStoreName(dataStreamName, 1, epochMillis), + DataStream.getDefaultFailureStoreName(dataStreamName, 2, epochMillis), + otherIndex.getIndex().getName() + ) + ); + } + // Test default with wildcard expression { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; @@ -2832,7 +2869,7 @@ public void testDataStreamsWithFailureStore() { // Test explicit include failure store with wildcard expression { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; - Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*::*"); + Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*::data", "my-*::failures"); assertThat(result.length, equalTo(5)); List indexNames = Arrays.stream(result).map(Index::getName).toList(); assertThat( @@ -3225,8 +3262,8 @@ public void testDataStreamsNames() { assertThat(streams, containsInAnyOrder(new ResolvedExpression(dataStream1, DATA), new ResolvedExpression(dataStream2, DATA))); assertThat(names, containsInAnyOrder(dataStream1, dataStream2)); - streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*foobar::*"); - names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*foobar::*"); + streams = indexNameExpressionResolver.dataStreams(state, IndicesOptions.lenientExpand(), "*foobar::data", "*foobar::failures"); + names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*foobar::data", "*foobar::failures"); assertThat( streams, containsInAnyOrder( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 7cb13027a064d..d5cd14b7fd96f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -615,6 +615,7 @@ private static SystemDataStreamDescriptor systemDataStreamDescriptor() { .build(), Map.of(), List.of("stack"), + "stack", ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 3c8b6ebbfb271..7c65d859fbcf6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -9,8 +9,10 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ClusterSettings; @@ -21,11 +23,19 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInProgressException; +import org.elasticsearch.snapshots.SnapshotInfoTestUtils; +import org.elasticsearch.test.index.IndexVersionUtils; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateMapping; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -86,6 +96,65 @@ public void testAddBackingIndex() { IndexMetadata zeroIndex = newState.metadata().index(ds.getIndices().get(0)); assertThat(zeroIndex.getIndex(), equalTo(indexToAdd.getIndex())); assertThat(zeroIndex.getSettings().get("index.hidden"), equalTo("true")); + assertThat(zeroIndex.isSystem(), equalTo(false)); + assertThat(zeroIndex.getAliases().size(), equalTo(0)); + } + + public void testAddBackingIndexToSystemDataStream() { + final long epochMillis = System.currentTimeMillis(); + final int numBackingIndices = randomIntBetween(1, 4); + final String dataStreamName = randomAlphaOfLength(5); + IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices]; + Metadata.Builder mb = Metadata.builder(); + for (int k = 0; k < numBackingIndices; k++) { + backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis)) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp")) + .system(true) + .build(); + mb.put(backingIndices[k], false); + } + + DataStream dataStream = DataStream.builder(dataStreamName, Arrays.stream(backingIndices).map(IndexMetadata::getIndex).toList()) + .setSystem(true) + .setHidden(true) + .build(); + mb.put(dataStream); + + final IndexMetadata indexToAdd = IndexMetadata.builder(randomAlphaOfLength(5)) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(generateMapping("@timestamp")) + .system(false) + .build(); + mb.put(indexToAdd, false); + + ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build(); + ClusterState newState = MetadataDataStreamsService.modifyDataStream( + originalState, + List.of(DataStreamAction.addBackingIndex(dataStreamName, indexToAdd.getIndex().getName())), + this::getMapperService, + Settings.EMPTY + ); + + IndexAbstraction ds = newState.metadata().getIndicesLookup().get(dataStreamName); + assertThat(ds, notNullValue()); + assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(ds.getIndices().size(), equalTo(numBackingIndices + 1)); + List backingIndexNames = ds.getIndices().stream().filter(x -> x.getName().startsWith(".ds-")).map(Index::getName).toList(); + assertThat( + backingIndexNames, + containsInAnyOrder( + Arrays.stream(backingIndices).map(IndexMetadata::getIndex).map(Index::getName).toList().toArray(Strings.EMPTY_ARRAY) + ) + ); + IndexMetadata zeroIndex = newState.metadata().index(ds.getIndices().get(0)); + assertThat(zeroIndex.getIndex(), equalTo(indexToAdd.getIndex())); + assertThat(zeroIndex.getSettings().get("index.hidden"), equalTo("true")); + assertThat(zeroIndex.isSystem(), equalTo(true)); assertThat(zeroIndex.getAliases().size(), equalTo(0)); } @@ -455,6 +524,55 @@ public void testUpdateDataStreamOptions() { assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY)); } + public void testDeleteMissing() { + DataStream dataStream = DataStreamTestHelper.randomInstance(); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build(); + + ResourceNotFoundException e = expectThrows( + ResourceNotFoundException.class, + () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY) + ); + assertThat(e.getMessage(), containsString(dataStream.getName())); + } + + public void testDeleteSnapshotting() { + String dataStreamName = randomAlphaOfLength(5); + Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); + SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry( + SnapshotsInProgress.Entry.snapshot( + snapshot, + true, + false, + SnapshotsInProgress.State.INIT, + Collections.emptyMap(), + List.of(dataStreamName), + Collections.emptyList(), + System.currentTimeMillis(), + (long) randomIntBetween(0, 1000), + Map.of(), + null, + SnapshotInfoTestUtils.randomUserMetadata(), + IndexVersionUtils.randomVersion() + ) + ); + final DataStream dataStream = DataStreamTestHelper.randomInstance(dataStreamName); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT) + .putCustom(SnapshotsInProgress.TYPE, snaps) + .metadata(Metadata.builder().put(dataStream).build()) + .build(); + Exception e = expectThrows( + SnapshotInProgressException.class, + () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY) + ); + assertEquals( + "Cannot delete data streams that are being snapshotted: [" + + dataStreamName + + "]. Try again after snapshot finishes " + + "or cancel the currently running snapshot.", + e.getMessage() + ); + } + private MapperService getMapperService(IndexMetadata im) { try { String mapping = im.mapping().source().toString(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 85dd788641367..1365f56b50e5f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -2593,6 +2593,59 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { ); } + public void testComposableTemplateWithSubobjectsFalseObjectAndSubfield() throws Exception { + MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + ComponentTemplate subobjects = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "properties": { + "foo": { + "type": "object", + "subobjects": false + }, + "foo.bar": { + "type": "keyword" + } + } + } + """), null), null, null); + + state = service.addComponentTemplate(state, true, "subobjects", subobjects); + ComposableIndexTemplate it = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(new Template(null, null, null)) + .componentTemplates(List.of("subobjects", "field_mapping")) + .priority(0L) + .version(1L) + .build(); + state = service.addIndexTemplateV2(state, true, "composable-template", it); + + List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(1)); + List> parsedMappings = mappings.stream().map(m -> { + try { + return MapperService.parseMapping(NamedXContentRegistry.EMPTY, m); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }).toList(); + + assertThat( + parsedMappings.get(0), + equalTo( + Map.of( + "_doc", + Map.of("properties", Map.of("foo.bar", Map.of("type", "keyword"), "foo", Map.of("type", "object", "subobjects", false))) + ) + ) + ); + } + public void testAddIndexTemplateWithDeprecatedComponentTemplate() throws Exception { ClusterState state = ClusterState.EMPTY_STATE; final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index 63e92835ba8db..71c39cdefc702 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -463,6 +463,7 @@ public void testSettingsVersion() throws IOException { mapperSupplier, removeAlias, failureStore, + false, nodeSettings ); Metadata metadata = metadataBuilder.build(); @@ -489,6 +490,7 @@ public void testSettingsVersion() throws IOException { mapperSupplier, removeAlias, failureStore, + false, nodeSettings ); Metadata metadata = metadataBuilder.build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index be60f5246b42b..42fe3be04da7e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -48,7 +48,6 @@ import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; -import org.elasticsearch.upgrades.FeatureMigrationResults; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -2287,8 +2286,6 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata ); } else if (custom instanceof DesiredNodesMetadata) { chunkCount += checkChunkSize(custom, params, 1); - } else if (custom instanceof FeatureMigrationResults featureMigrationResults) { - chunkCount += checkChunkSize(custom, params, 2 + featureMigrationResults.getFeatureStatuses().size()); } else if (custom instanceof IndexGraveyard indexGraveyard) { chunkCount += checkChunkSize(custom, params, 2 + indexGraveyard.getTombstones().size()); } else if (custom instanceof IngestMetadata ingestMetadata) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java index 2bf34dcfd2a34..dd3876afd3c74 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.action.support.IndexComponentSelector.ALL_APPLICABLE; import static org.elasticsearch.action.support.IndexComponentSelector.DATA; import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES; import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; @@ -38,7 +37,6 @@ public void testResolveExpression() { assertThat(resolve(selectorsAllowed, "testXXX"), equalTo(new ResolvedExpression("testXXX", DATA))); assertThat(resolve(selectorsAllowed, "testXXX::data"), equalTo(new ResolvedExpression("testXXX", DATA))); assertThat(resolve(selectorsAllowed, "testXXX::failures"), equalTo(new ResolvedExpression("testXXX", FAILURES))); - assertThat(resolve(selectorsAllowed, "testXXX::*"), equalTo(new ResolvedExpression("testXXX", ALL_APPLICABLE))); // Disallow selectors (example: creating, modifying, or deleting indices/data streams/aliases). // Accepts standard expressions but throws when selectors are specified. @@ -47,7 +45,6 @@ public void testResolveExpression() { assertThat(resolve(noSelectors, "testXXX"), equalTo(new ResolvedExpression("testXXX"))); expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::data")); expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::failures")); - expectThrows(IllegalArgumentException.class, () -> resolve(noSelectors, "testXXX::*")); // === Errors // Only recognized components can be selected @@ -116,9 +113,7 @@ public void testCombineExpressionWithSelector() { assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", null), is(equalTo("a"))); assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", ""), is(equalTo("a::"))); assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", "b"), is(equalTo("a::b"))); - assertThat(IndexNameExpressionResolver.combineSelectorExpression("a", "*"), is(equalTo("a::*"))); assertThat(IndexNameExpressionResolver.combineSelectorExpression("*", "b"), is(equalTo("*::b"))); - assertThat(IndexNameExpressionResolver.combineSelectorExpression("*", "*"), is(equalTo("*::*"))); } public void testHasSelectorSuffix() { @@ -151,14 +146,14 @@ public void testSplitSelectorExpression() { assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::data"), is(equalTo(new Tuple<>("a", "data")))); assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::failures"), is(equalTo(new Tuple<>("a", "failures")))); - assertThat(IndexNameExpressionResolver.splitSelectorExpression("a::*"), is(equalTo(new Tuple<>("a", "*")))); + expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::*")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::random")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::d*ta")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::*ailures")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("a::**")); expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("index::data::*")); - assertThat(IndexNameExpressionResolver.splitSelectorExpression("::*"), is(equalTo(new Tuple<>("", "*")))); + expectThrows(InvalidIndexNameException.class, () -> IndexNameExpressionResolver.splitSelectorExpression("::*")); } private static IndicesOptions getOptionsForSelectors() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java index ac9d5021329ab..04fd14ce581f0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java @@ -11,20 +11,32 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SystemIndexMetadataUpgradeServiceTests extends ESTestCase { @@ -50,17 +62,63 @@ public class SystemIndexMetadataUpgradeServiceTests extends ESTestCase { .setOrigin("FAKE_ORIGIN") .build(); + private static final String SYSTEM_DATA_STREAM_NAME = ".my-ds"; + private static final String SYSTEM_DATA_STREAM_INDEX_NAME = DataStream.BACKING_INDEX_PREFIX + SYSTEM_DATA_STREAM_NAME + "-1"; + private static final String SYSTEM_DATA_STREAM_FAILSTORE_NAME = DataStream.FAILURE_STORE_PREFIX + SYSTEM_DATA_STREAM_NAME; + private static final SystemDataStreamDescriptor SYSTEM_DATA_STREAM_DESCRIPTOR = new SystemDataStreamDescriptor( + SYSTEM_DATA_STREAM_NAME, + "System datastream for test", + SystemDataStreamDescriptor.Type.INTERNAL, + ComposableIndexTemplate.builder().build(), + Collections.emptyMap(), + Collections.singletonList("FAKE_ORIGIN"), + "FAKE_ORIGIN", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ); + private SystemIndexMetadataUpgradeService service; + private ClusterStateTaskListener task; + private ClusterStateTaskExecutor executor; + @SuppressWarnings("unchecked") @Before public void setUpTest() { // set up a system index upgrade service + ClusterService clusterService = mock(ClusterService.class); + MasterServiceTaskQueue queue = mock(MasterServiceTaskQueue.class); + when(clusterService.createTaskQueue(eq("system-indices-metadata-upgrade"), eq(Priority.NORMAL), any())).thenAnswer(invocation -> { + executor = invocation.getArgument(2, ClusterStateTaskExecutor.class); + return queue; + }); + doAnswer(invocation -> { + task = invocation.getArgument(1, ClusterStateTaskListener.class); + return null; + }).when(queue).submitTask(any(), any(), any()); + this.service = new SystemIndexMetadataUpgradeService( - new SystemIndices(List.of(new SystemIndices.Feature("foo", "a test feature", List.of(DESCRIPTOR)))), - mock(ClusterService.class) + new SystemIndices( + List.of( + new SystemIndices.Feature("foo", "a test feature", List.of(DESCRIPTOR)), + new SystemIndices.Feature( + "sds", + "system data stream feature", + Collections.emptyList(), + Collections.singletonList(SYSTEM_DATA_STREAM_DESCRIPTOR) + ) + ) + ), + clusterService ); } + private ClusterState executeTask(ClusterState clusterState) { + try { + return ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, Collections.singletonList(task)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + /** * When we upgrade Elasticsearch versions, existing indices may be newly * defined as system indices. If such indices are set without "hidden," we need @@ -76,6 +134,54 @@ public void testUpgradeVisibleIndexToSystemIndex() throws Exception { assertSystemUpgradeAppliesHiddenSetting(hiddenIndexMetadata); } + public void testUpgradeDataStreamToSystemDataStream() { + IndexMetadata dsIndexMetadata = IndexMetadata.builder(SYSTEM_DATA_STREAM_INDEX_NAME) + .system(false) + .settings(getSettingsBuilder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true)) + .build(); + IndexMetadata fsIndexMetadata = IndexMetadata.builder(SYSTEM_DATA_STREAM_FAILSTORE_NAME) + .system(false) + .settings(getSettingsBuilder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true)) + .build(); + DataStream.DataStreamIndices failureIndices = DataStream.DataStreamIndices.failureIndicesBuilder( + Collections.singletonList(fsIndexMetadata.getIndex()) + ).build(); + DataStream dataStream = DataStream.builder(SYSTEM_DATA_STREAM_NAME, Collections.singletonList(dsIndexMetadata.getIndex())) + .setFailureIndices(failureIndices) + .setHidden(false) + .setSystem(false) + .build(); + + assertTrue(dataStream.containsIndex(dsIndexMetadata.getIndex().getName())); + assertTrue(dataStream.containsIndex(fsIndexMetadata.getIndex().getName())); + + Metadata.Builder clusterMetadata = new Metadata.Builder(); + clusterMetadata.put(dataStream); + clusterMetadata.put(dsIndexMetadata, true); + clusterMetadata.put(fsIndexMetadata, true); + + ClusterState clusterState = ClusterState.builder(new ClusterName("system-index-metadata-upgrade-service-tests")) + .metadata(clusterMetadata.build()) + .customs(Map.of()) + .build(); + + service.submitUpdateTask(Collections.emptyList(), Collections.singletonList(dataStream)); + // Execute a metadata upgrade task on the initial cluster state + ClusterState newState = executeTask(clusterState); + + DataStream updatedDataStream = newState.metadata().dataStreams().get(dataStream.getName()); + assertThat(updatedDataStream.isSystem(), equalTo(true)); + assertThat(updatedDataStream.isHidden(), equalTo(true)); + + IndexMetadata updatedIndexMetadata = newState.metadata().index(dsIndexMetadata.getIndex().getName()); + assertThat(updatedIndexMetadata.isSystem(), equalTo(true)); + assertThat(updatedIndexMetadata.isHidden(), equalTo(true)); + + IndexMetadata updatedFailstoreMetadata = newState.metadata().index(fsIndexMetadata.getIndex().getName()); + assertThat(updatedFailstoreMetadata.isSystem(), equalTo(true)); + assertThat(updatedFailstoreMetadata.isHidden(), equalTo(true)); + } + /** * If a system index erroneously is set to visible, we should remedy that situation. */ @@ -210,7 +316,7 @@ public void testIsVisible() { assertThat(service.requiresUpdate(systemVisibleIndex), equalTo(true)); } - private void assertSystemUpgradeAppliesHiddenSetting(IndexMetadata hiddenIndexMetadata) throws Exception { + private void assertSystemUpgradeAppliesHiddenSetting(IndexMetadata hiddenIndexMetadata) { assertTrue("Metadata should require update but does not", service.requiresUpdate(hiddenIndexMetadata)); Metadata.Builder clusterMetadata = new Metadata.Builder(); clusterMetadata.put(IndexMetadata.builder(hiddenIndexMetadata)); @@ -220,8 +326,9 @@ private void assertSystemUpgradeAppliesHiddenSetting(IndexMetadata hiddenIndexMe .customs(Map.of()) .build(); + service.submitUpdateTask(Collections.singletonList(hiddenIndexMetadata.getIndex()), Collections.emptyList()); // Get a metadata upgrade task and execute it on the initial cluster state - ClusterState newState = service.getTask().execute(clusterState); + ClusterState newState = executeTask(clusterState); IndexMetadata result = newState.metadata().index(SYSTEM_INDEX_NAME); assertThat(result.isSystem(), equalTo(true)); @@ -238,8 +345,9 @@ private void assertSystemUpgradeHidesAlias(IndexMetadata visibleAliasMetadata) t .customs(Map.of()) .build(); + service.submitUpdateTask(Collections.singletonList(visibleAliasMetadata.getIndex()), Collections.emptyList()); // Get a metadata upgrade task and execute it on the initial cluster state - ClusterState newState = service.getTask().execute(clusterState); + ClusterState newState = executeTask(clusterState); IndexMetadata result = newState.metadata().index(SYSTEM_INDEX_NAME); assertThat(result.isSystem(), equalTo(true)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index 9d9a5ebd37218..1eeaef473521d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -25,7 +25,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.elasticsearch.action.support.IndexComponentSelector.ALL_APPLICABLE; import static org.elasticsearch.action.support.IndexComponentSelector.DATA; import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; @@ -54,19 +53,19 @@ public void testConvertWildcardsJustIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*", DATA)), equalTo(resolvedExpressionsSet("kuku")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY", "kuku")) ); } @@ -87,7 +86,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); context = new IndexNameExpressionResolver.Context( @@ -96,7 +95,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXYY")) ); context = new IndexNameExpressionResolver.Context( @@ -105,7 +104,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXXY")) ); } @@ -128,31 +127,27 @@ public void testMultipleWildcards() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*", DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); assertThat( - newHashSet( - IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y", ALL_APPLICABLE) - ), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y", DATA)), equalTo(resolvedExpressionsSet("testXXY", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*", DATA)), equalTo(resolvedExpressionsSet("kukuYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*", ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*", DATA)), equalTo(resolvedExpressionsSet("testXXY", "testXYY", "testYYY", "kukuYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X", ALL_APPLICABLE)) - .size(), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X", DATA)).size(), equalTo(0) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*X", ALL_APPLICABLE)) - .size(), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*X", DATA)).size(), equalTo(0) ); } @@ -171,7 +166,7 @@ public void testAll() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); } @@ -189,7 +184,7 @@ public void testAll() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, null)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(resolvedExpressionsNoSelectorSet("testXXX", "testXYY", "testYYY")) ); } @@ -212,10 +207,7 @@ public void testAllAliases() { IndicesOptions.lenientExpandOpen(), // don't include hidden SystemIndexAccessLevel.NONE ); - assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), - equalTo(newHashSet()) - ); + assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(newHashSet())); } { @@ -235,7 +227,7 @@ public void testAllAliases() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(resolvedExpressionsSet("index-visible-alias")) ); } @@ -290,13 +282,8 @@ public void testAllDataStreams() { equalTo(resolvedExpressionsSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis))) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), - equalTo( - resolvedExpressionsSet( - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis) - ) - ) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, FAILURES)), + equalTo(resolvedExpressionsSet(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis))) ); } @@ -328,10 +315,7 @@ public void testAllDataStreams() { ); assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, DATA)), equalTo(Set.of())); - assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, ALL_APPLICABLE)), - equalTo(Set.of()) - ); + assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context, FAILURES)), equalTo(Set.of())); } } @@ -455,7 +439,7 @@ public void testResolveAliases() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo_a*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_index", DATA), new ResolvedExpression("bar_index", DATA))); } @@ -463,7 +447,7 @@ public void testResolveAliases() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesLenientContext, "foo_a*", - ALL_APPLICABLE + DATA ); assertEquals(0, indices.size()); } @@ -471,7 +455,7 @@ public void testResolveAliases() { Set indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesStrictContext, "foo_a*", - ALL_APPLICABLE + DATA ); assertThat(indices, empty()); } @@ -479,7 +463,7 @@ public void testResolveAliases() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -494,7 +478,7 @@ public void testResolveAliases() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesLenientContext, "foo*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA))); } @@ -502,7 +486,7 @@ public void testResolveAliases() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesStrictContext, "foo*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA))); } @@ -556,7 +540,7 @@ public void testResolveDataStreams() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo_*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -571,7 +555,7 @@ public void testResolveDataStreams() { indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "bar_*", - ALL_APPLICABLE + DATA ); assertThat(indices, containsInAnyOrder(new ResolvedExpression("bar_bar", DATA), new ResolvedExpression("bar_index", DATA))); } @@ -602,7 +586,7 @@ public void testResolveDataStreams() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesAndDataStreamsContext, "foo_*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -611,9 +595,7 @@ public void testResolveDataStreams() { new ResolvedExpression("bar_index", DATA), new ResolvedExpression("foo_foo", DATA), new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA) + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA) ) ); @@ -632,26 +614,6 @@ public void testResolveDataStreams() { ) ) ); - - // include all wildcard adds the data stream's backing indices - indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAliasesAndDataStreamsContext, - "*", - ALL_APPLICABLE - ); - assertThat( - indices, - containsInAnyOrder( - new ResolvedExpression("foo_index", DATA), - new ResolvedExpression("bar_index", DATA), - new ResolvedExpression("foo_foo", DATA), - new ResolvedExpression("bar_bar", DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA) - ) - ); } { @@ -681,7 +643,7 @@ public void testResolveDataStreams() { Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesDataStreamsAndHiddenIndices, "foo_*", - ALL_APPLICABLE + DATA ); assertThat( indices, @@ -690,9 +652,7 @@ public void testResolveDataStreams() { new ResolvedExpression("bar_index", DATA), new ResolvedExpression("foo_foo", DATA), new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), DATA), - new ResolvedExpression(DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis), DATA) + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), DATA) ) ); @@ -712,32 +672,11 @@ public void testResolveDataStreams() { ) ); - // Resolve both backing and failure indices - indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAliasesDataStreamsAndHiddenIndices, - "foo_*", - ALL_APPLICABLE - ); - assertThat( - newHashSet(indices), - equalTo( - resolvedExpressionsSet( - "foo_index", - "bar_index", - "foo_foo", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis) - ) - ) - ); - // include all wildcard adds the data stream's backing indices indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesDataStreamsAndHiddenIndices, "*", - ALL_APPLICABLE + DATA ); assertThat( newHashSet(indices), @@ -770,28 +709,6 @@ public void testResolveDataStreams() { ) ) ); - - // include all wildcard adds the data stream's backing and failure indices - indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAliasesDataStreamsAndHiddenIndices, - "*", - ALL_APPLICABLE - ); - assertThat( - newHashSet(indices), - equalTo( - resolvedExpressionsSet( - "foo_index", - "bar_index", - "foo_foo", - "bar_bar", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 1, epochMillis), - DataStream.getDefaultFailureStoreName("foo_logs", 2, epochMillis) - ) - ) - ); } } @@ -824,7 +741,7 @@ public void testMatchesConcreteIndicesWildcardAndAliases() { Collection matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "*", - ALL_APPLICABLE + DATA ); assertThat( matches, @@ -835,7 +752,7 @@ public void testMatchesConcreteIndicesWildcardAndAliases() { new ResolvedExpression("bar_index", DATA) ) ); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*", ALL_APPLICABLE); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*", DATA); assertThat( matches, containsInAnyOrder( @@ -845,11 +762,7 @@ public void testMatchesConcreteIndicesWildcardAndAliases() { new ResolvedExpression("bar_index", DATA) ) ); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - indicesAndAliasesContext, - "foo*", - ALL_APPLICABLE - ); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(indicesAndAliasesContext, "foo*", DATA); assertThat( matches, containsInAnyOrder( @@ -858,11 +771,7 @@ public void testMatchesConcreteIndicesWildcardAndAliases() { new ResolvedExpression("bar_index", DATA) ) ); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( - onlyIndicesContext, - "foo*", - ALL_APPLICABLE - ); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "foo*", DATA); assertThat(matches, containsInAnyOrder(new ResolvedExpression("foo_foo", DATA), new ResolvedExpression("foo_index", DATA))); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java index 586f088d554a5..5eea25d85865b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java @@ -1922,6 +1922,7 @@ private SystemIndices getSystemIndices( .build(), Map.of(), List.of("test"), + "test", new ExecutorNames( ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, diff --git a/server/src/test/java/org/elasticsearch/common/StringsTests.java b/server/src/test/java/org/elasticsearch/common/StringsTests.java index cc2927a34990d..36ab4438dd487 100644 --- a/server/src/test/java/org/elasticsearch/common/StringsTests.java +++ b/server/src/test/java/org/elasticsearch/common/StringsTests.java @@ -228,51 +228,47 @@ public void testDelimitedListToStringArray() { public void testCollectionToDelimitedStringWithLimitZero() { final String delimiter = randomFrom("", ",", ", ", "/"); - final String prefix = randomFrom("", "["); - final String suffix = randomFrom("", "]"); final int count = between(0, 100); final List strings = new ArrayList<>(count); while (strings.size() < count) { // avoid starting with a sequence of empty appends, it makes the assertions much messier - final int minLength = strings.isEmpty() && delimiter.isEmpty() && prefix.isEmpty() && suffix.isEmpty() ? 1 : 0; + final int minLength = strings.isEmpty() && delimiter.isEmpty() ? 1 : 0; strings.add(randomAlphaOfLength(between(minLength, 10))); } final StringBuilder stringBuilder = new StringBuilder(); - collectionToDelimitedStringWithLimit(strings, delimiter, prefix, suffix, 0, stringBuilder); + collectionToDelimitedStringWithLimit(strings, delimiter, 0, stringBuilder); final String completelyTruncatedDescription = stringBuilder.toString(); if (count == 0) { assertThat(completelyTruncatedDescription, equalTo("")); } else if (count == 1) { - assertThat(completelyTruncatedDescription, equalTo(prefix + strings.get(0) + suffix)); + assertThat(completelyTruncatedDescription, equalTo(strings.get(0))); } else { assertThat( completelyTruncatedDescription, - equalTo(prefix + strings.get(0) + suffix + delimiter + "... (" + count + " in total, " + (count - 1) + " omitted)") + equalTo(strings.get(0) + delimiter + "... (" + count + " in total, " + (count - 1) + " omitted)") ); } } public void testCollectionToDelimitedStringWithLimitTruncation() { final String delimiter = randomFrom("", ",", ", ", "/"); - final String prefix = randomFrom("", "["); - final String suffix = randomFrom("", "]"); final int count = between(2, 100); final List strings = new ArrayList<>(count); while (strings.size() < count) { // avoid empty appends, it makes the assertions much messier - final int minLength = delimiter.isEmpty() && prefix.isEmpty() && suffix.isEmpty() ? 1 : 0; + final int minLength = delimiter.isEmpty() ? 1 : 0; strings.add(randomAlphaOfLength(between(minLength, 10))); } - final int fullDescriptionLength = collectionToDelimitedString(strings, delimiter, prefix, suffix).length(); - final int lastItemSize = prefix.length() + strings.get(count - 1).length() + suffix.length(); + final int fullDescriptionLength = collectionToDelimitedString(strings, delimiter).length(); + final int lastItemSize = strings.get(count - 1).length(); final int truncatedLength = between(0, fullDescriptionLength - lastItemSize - 1); final StringBuilder stringBuilder = new StringBuilder(); - collectionToDelimitedStringWithLimit(strings, delimiter, prefix, suffix, truncatedLength, stringBuilder); + collectionToDelimitedStringWithLimit(strings, delimiter, truncatedLength, stringBuilder); final String truncatedDescription = stringBuilder.toString(); assertThat(truncatedDescription, allOf(containsString("... (" + count + " in total,"), endsWith(" omitted)"))); @@ -280,14 +276,12 @@ public void testCollectionToDelimitedStringWithLimitTruncation() { assertThat( truncatedDescription, truncatedDescription.length(), - lessThanOrEqualTo(truncatedLength + (prefix + "0123456789" + suffix + delimiter + "... (999 in total, 999 omitted)").length()) + lessThanOrEqualTo(truncatedLength + ("0123456789" + delimiter + "... (999 in total, 999 omitted)").length()) ); } public void testCollectionToDelimitedStringWithLimitNoTruncation() { final String delimiter = randomFrom("", ",", ", ", "/"); - final String prefix = randomFrom("", "["); - final String suffix = randomFrom("", "]"); final int count = between(1, 100); final List strings = new ArrayList<>(count); @@ -295,17 +289,17 @@ public void testCollectionToDelimitedStringWithLimitNoTruncation() { strings.add(randomAlphaOfLength(between(0, 10))); } - final String fullDescription = collectionToDelimitedString(strings, delimiter, prefix, suffix); + final String fullDescription = collectionToDelimitedString(strings, delimiter); for (String string : strings) { - assertThat(fullDescription, containsString(prefix + string + suffix)); + assertThat(fullDescription, containsString(string)); } - final int lastItemSize = prefix.length() + strings.get(count - 1).length() + suffix.length(); + final int lastItemSize = strings.get(count - 1).length(); final int minLimit = fullDescription.length() - lastItemSize; final int limit = randomFrom(between(minLimit, fullDescription.length()), between(minLimit, Integer.MAX_VALUE), Integer.MAX_VALUE); final StringBuilder stringBuilder = new StringBuilder(); - collectionToDelimitedStringWithLimit(strings, delimiter, prefix, suffix, limit, stringBuilder); + collectionToDelimitedStringWithLimit(strings, delimiter, limit, stringBuilder); assertThat(stringBuilder.toString(), equalTo(fullDescription)); } diff --git a/server/src/test/java/org/elasticsearch/common/cli/EnvironmentAwareCommandTests.java b/server/src/test/java/org/elasticsearch/common/cli/EnvironmentAwareCommandTests.java index 0295cb731a587..0a3e2cdf9bb5e 100644 --- a/server/src/test/java/org/elasticsearch/common/cli/EnvironmentAwareCommandTests.java +++ b/server/src/test/java/org/elasticsearch/common/cli/EnvironmentAwareCommandTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cli.CommandTestCase; import org.elasticsearch.cli.ProcessInfo; import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.junit.Before; @@ -127,4 +128,15 @@ public void testDockerEnvVarSettingsOverrideCommandLine() throws Exception { }; execute("-Esimple.setting=original"); } + + public void testDuplicateCommandLineSetting() { + var e = expectThrows(UserException.class, () -> execute("-E", "my.setting=foo", "-E", "my.setting=bar")); + assertThat(e.getMessage(), equalTo("setting [my.setting] set twice via command line -E")); + } + + public void testConflictingPathCommandLineSettingWithSysprop() { + sysprops.put("es.path.data", "foo"); + var e = expectThrows(UserException.class, () -> execute("-E", "path.data=bar")); + assertThat(e.getMessage(), equalTo("setting [path.data] found via command-line -E and system property [es.path.data]")); + } } diff --git a/server/src/test/java/org/elasticsearch/common/file/AbstractFileWatchingServiceTests.java b/server/src/test/java/org/elasticsearch/common/file/AbstractFileWatchingServiceTests.java index ebf1064c2ae3f..1240be169fe44 100644 --- a/server/src/test/java/org/elasticsearch/common/file/AbstractFileWatchingServiceTests.java +++ b/server/src/test/java/org/elasticsearch/common/file/AbstractFileWatchingServiceTests.java @@ -24,12 +24,14 @@ import org.junit.Before; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardWatchEventKinds; import java.nio.file.WatchKey; +import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Instant; import java.time.LocalDateTime; @@ -38,6 +40,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.sameInstance; @@ -81,6 +84,39 @@ protected void processInitialFileMissing() { countDownLatch.countDown(); } } + + // the following methods are a workaround to ensure exclusive access for files + // required by child watchers; this is required because we only check the caller's module + // not the entire stack + @Override + protected boolean filesExists(Path path) { + return Files.exists(path); + } + + @Override + protected boolean filesIsDirectory(Path path) { + return Files.isDirectory(path); + } + + @Override + protected A filesReadAttributes(Path path, Class clazz) throws IOException { + return Files.readAttributes(path, clazz); + } + + @Override + protected Stream filesList(Path dir) throws IOException { + return Files.list(dir); + } + + @Override + protected Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException { + return Files.setLastModifiedTime(path, time); + } + + @Override + protected InputStream filesNewInputStream(Path path) throws IOException { + return Files.newInputStream(path); + } } private AbstractFileWatchingService fileWatchingService; @@ -104,7 +140,7 @@ public void setUp() throws Exception { env = newEnvironment(Settings.EMPTY); - Files.createDirectories(env.configFile()); + Files.createDirectories(env.configDir()); fileWatchingService = new TestFileWatchingService(getWatchedFilePath(env)); } @@ -203,7 +239,7 @@ private void writeTestFile(Path path, String contents) throws IOException { } private static Path getWatchedFilePath(Environment env) { - return env.configFile().toAbsolutePath().resolve("test").resolve("test.json"); + return env.configDir().toAbsolutePath().resolve("test").resolve("test.json"); } } diff --git a/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java b/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java index 0dd1db64c144f..0c2dc68a01464 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java @@ -97,7 +97,7 @@ public void testCreate() { } public void testProcessSettingsFile() throws Exception { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSON); LocallyMountedSecrets secrets = new LocallyMountedSecrets(env); assertTrue(secrets.isLoaded()); assertThat(secrets.getVersion(), equalTo(1L)); @@ -109,7 +109,7 @@ public void testProcessSettingsFile() throws Exception { } public void testProcessDeprecatedSettingsFile() throws Exception { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSONDepricated); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSONDepricated); LocallyMountedSecrets secrets = new LocallyMountedSecrets(env); assertTrue(secrets.isLoaded()); assertThat(secrets.getVersion(), equalTo(1L)); @@ -119,7 +119,7 @@ public void testProcessDeprecatedSettingsFile() throws Exception { } public void testDuplicateSettingKeys() throws Exception { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSONDuplicateKeys); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSONDuplicateKeys); Exception e = expectThrows(Exception.class, () -> new LocallyMountedSecrets(env)); assertThat(e, instanceOf(XContentParseException.class)); assertThat(e.getMessage(), containsString("failed to parse field")); @@ -134,7 +134,7 @@ public void testDuplicateSettingKeys() throws Exception { } public void testSettingsGetFile() throws IOException, GeneralSecurityException { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSON); LocallyMountedSecrets secrets = new LocallyMountedSecrets(env); assertTrue(secrets.isLoaded()); assertThat(secrets.getSettingNames(), containsInAnyOrder("aaa", "ccc", "eee")); @@ -165,7 +165,7 @@ public void testSettingsGetFile() throws IOException, GeneralSecurityException { } public void testSettingsSHADigest() throws IOException, GeneralSecurityException { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSON); LocallyMountedSecrets secrets = new LocallyMountedSecrets(env); assertTrue(secrets.isLoaded()); assertThat(secrets.getSettingNames(), containsInAnyOrder("aaa", "ccc", "eee")); @@ -178,7 +178,7 @@ public void testSettingsSHADigest() throws IOException, GeneralSecurityException } public void testProcessBadSettingsFile() throws IOException { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), noMetadataJSON); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), noMetadataJSON); assertThat( expectThrows(IllegalArgumentException.class, () -> new LocallyMountedSecrets(env)).getMessage(), containsString("Required [metadata]") @@ -186,7 +186,7 @@ public void testProcessBadSettingsFile() throws IOException { } public void testSerializationWithSecrets() throws Exception { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSON); LocallyMountedSecrets secrets = new LocallyMountedSecrets(env); final BytesStreamOutput out = new BytesStreamOutput(); @@ -213,7 +213,7 @@ public void testSerializationNewlyCreated() throws Exception { } public void testClose() throws IOException { - writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON); + writeTestFile(env.configDir().resolve("secrets").resolve("secrets.json"), testJSON); LocallyMountedSecrets secrets = new LocallyMountedSecrets(env); assertEquals("bbb", secrets.getString("aaa").toString()); assertEquals("ddd", secrets.getString("ccc").toString()); diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java index cfdc5e6befaaa..d608136fa564e 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java @@ -44,6 +44,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.is; @@ -710,4 +711,22 @@ public void testProcessSetting() throws IOException { {"ant.bee":{"cat.dog":{"ewe":"value3"},"cat":"value2"},"ant":"value1"}""", Strings.toString(builder)); } + public void testGlobValues() throws IOException { + Settings test = Settings.builder().put("foo.x.bar", "1").build(); + + // no values + assertThat(test.getValues("foo.*.baz").toList(), empty()); + assertThat(test.getValues("fuz.*.bar").toList(), empty()); + + var values = test.getValues("foo.*.bar").toList(); + assertThat(values, containsInAnyOrder("1")); + + test = Settings.builder().put("foo.x.bar", "1").put("foo.y.bar", "2").build(); + values = test.getValues("foo.*.bar").toList(); + assertThat(values, containsInAnyOrder("1", "2")); + + values = test.getValues("foo.x.bar").toList(); + assertThat(values, contains("1")); + } + } diff --git a/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java b/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java index 32a8de20df9aa..0874a106e59e7 100644 --- a/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java +++ b/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.test.ESTestCase; +import static org.hamcrest.Matchers.equalTo; + public class SizeLimitingStringWriterTests extends ESTestCase { public void testSizeIsLimited() { SizeLimitingStringWriter writer = new SizeLimitingStringWriter(10); @@ -26,4 +28,11 @@ public void testSizeIsLimited() { expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append("a")); expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append("a", 0, 1)); } + + public void testLimitMessage() { + SizeLimitingStringWriter writer = new SizeLimitingStringWriter(3); + + var e = expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.write("abcdefgh")); + assertThat(e.getMessage(), equalTo("String [abc...] has size [8] which exceeds the size limit [3]")); + } } diff --git a/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java b/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java index e15bbbf75a529..08da6f0dfc957 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java @@ -27,6 +27,7 @@ import static org.elasticsearch.common.time.DateUtils.compareNanosToMillis; import static org.elasticsearch.common.time.DateUtils.toInstant; import static org.elasticsearch.common.time.DateUtils.toLong; +import static org.elasticsearch.common.time.DateUtils.toLongMillis; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.common.time.DateUtils.toNanoSeconds; import static org.hamcrest.Matchers.containsString; @@ -93,6 +94,44 @@ public void testInstantToLongMax() { assertThat(e.getMessage(), containsString("is after")); } + public void testInstantToLongMillis() { + assertThat(toLongMillis(Instant.EPOCH), is(0L)); + + Instant instant = createRandomInstant(); + long timeSinceEpochInMillis = instant.toEpochMilli(); + assertThat(toLongMillis(instant), is(timeSinceEpochInMillis)); + + Instant maxInstant = Instant.ofEpochSecond(Long.MAX_VALUE / 1000); + long maxInstantMillis = maxInstant.toEpochMilli(); + assertThat(toLongMillis(maxInstant), is(maxInstantMillis)); + + Instant minInstant = Instant.ofEpochSecond(Long.MIN_VALUE / 1000); + long minInstantMillis = minInstant.toEpochMilli(); + assertThat(toLongMillis(minInstant), is(minInstantMillis)); + } + + public void testInstantToLongMillisMin() { + /* negative millisecond value of this instant exceeds the maximum value a java long variable can store */ + Instant tooEarlyInstant = Instant.MIN; + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> toLongMillis(tooEarlyInstant)); + assertThat(e.getMessage(), containsString("too far in the past")); + + Instant tooEarlyInstant2 = Instant.ofEpochSecond(Long.MIN_VALUE / 1000 - 1); + e = expectThrows(IllegalArgumentException.class, () -> toLongMillis(tooEarlyInstant2)); + assertThat(e.getMessage(), containsString("too far in the past")); + } + + public void testInstantToLongMillisMax() { + /* millisecond value of this instant exceeds the maximum value a java long variable can store */ + Instant tooLateInstant = Instant.MAX; + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> toLongMillis(tooLateInstant)); + assertThat(e.getMessage(), containsString("too far in the future")); + + Instant tooLateInstant2 = Instant.ofEpochSecond(Long.MAX_VALUE / 1000 + 1); + e = expectThrows(IllegalArgumentException.class, () -> toLongMillis(tooLateInstant2)); + assertThat(e.getMessage(), containsString("too far in the future")); + } + public void testLongToInstant() { assertThat(toInstant(0), is(Instant.EPOCH)); assertThat(toInstant(1), is(Instant.EPOCH.plusNanos(1))); diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java index 2867c9e007937..e87c0d00c15cd 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Processors; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -754,4 +755,153 @@ public void onRejection(Exception e) { executor.execute(shouldBeRejected); assertTrue(rejected.get()); } + + public void testScalingWithEmptyCore() { + testScalingWithEmptyCore( + EsExecutors.newScaling( + getTestName(), + 0, + 1, + 0, + TimeUnit.MILLISECONDS, + true, + EsExecutors.daemonThreadFactory(getTestName()), + threadContext + ) + ); + } + + public void testScalingWithEmptyCoreAndKeepAlive() { + testScalingWithEmptyCore( + EsExecutors.newScaling( + getTestName(), + 0, + 1, + 1, + TimeUnit.MILLISECONDS, + true, + EsExecutors.daemonThreadFactory(getTestName()), + threadContext + ) + ); + } + + public void testScalingWithEmptyCoreAndLargerMaxSize() { + // TODO currently the reproduction of the starvation bug does not work if max pool size > 1 + // https://github.com/elastic/elasticsearch/issues/124867 + testScalingWithEmptyCore( + EsExecutors.newScaling( + getTestName(), + 0, + between(2, 5), + 0, + TimeUnit.MILLISECONDS, + true, + EsExecutors.daemonThreadFactory(getTestName()), + threadContext + ) + ); + } + + public void testScalingWithEmptyCoreAndKeepAliveAndLargerMaxSize() { + // TODO currently the reproduction of the starvation bug does not work if max pool size > 1 + // https://github.com/elastic/elasticsearch/issues/124867 + testScalingWithEmptyCore( + EsExecutors.newScaling( + getTestName(), + 0, + between(2, 5), + 1, + TimeUnit.MILLISECONDS, + true, + EsExecutors.daemonThreadFactory(getTestName()), + threadContext + ) + ); + } + + public void testScalingWithEmptyCoreAndWorkerPoolProbing() { + // https://github.com/elastic/elasticsearch/issues/124667 is difficult to reproduce if max pool size > 1. + // if probing mitigates the bug for max pool size = 1, we're good for larger pool sizes as well. + // the executor is created directly here, newScaling doesn't use ExecutorScalingQueue & probing if max pool size = 1. + testScalingWithEmptyCore( + new EsThreadPoolExecutor( + getTestName(), + 0, + 1, + 0, + TimeUnit.MILLISECONDS, + new EsExecutors.ExecutorScalingQueue<>(), + EsExecutors.daemonThreadFactory(getTestName()), + new EsExecutors.ForceQueuePolicy(true, true), + threadContext + ) + ); + } + + public void testScalingWithEmptyCoreAndKeepAliveAndWorkerPoolProbing() { + // https://github.com/elastic/elasticsearch/issues/124667 is difficult to reproduce if max pool size > 1. + // if probing mitigates the bug for max pool size = 1, we're good for larger pool sizes as well. + // the executor is created directly here, newScaling doesn't use ExecutorScalingQueue & probing if max pool size = 1. + testScalingWithEmptyCore( + new EsThreadPoolExecutor( + getTestName(), + 0, + 1, + 1, + TimeUnit.MILLISECONDS, + new EsExecutors.ExecutorScalingQueue<>(), + EsExecutors.daemonThreadFactory(getTestName()), + new EsExecutors.ForceQueuePolicy(true, true), + threadContext + ) + ); + } + + private void testScalingWithEmptyCore(EsThreadPoolExecutor executor) { + try { + class Task extends AbstractRunnable { + private int remaining; + private final CountDownLatch doneLatch; + + Task(int iterations, CountDownLatch doneLatch) { + this.remaining = iterations; + this.doneLatch = doneLatch; + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + if (--remaining == 0) { + doneLatch.countDown(); + } else { + logger.trace("--> remaining [{}]", remaining); + final long keepAliveNanos = executor.getKeepAliveTime(TimeUnit.NANOSECONDS); + new Thread(() -> { + if (keepAliveNanos > 0) { + final var targetNanoTime = System.nanoTime() + keepAliveNanos + between(-10_000, 10_000); + while (System.nanoTime() < targetNanoTime) { + Thread.yield(); + } + } + executor.execute(Task.this); + }).start(); + } + } + } + + for (int i = 0; i < 20; i++) { + logger.trace("--> attempt [{}]", i); + final var doneLatch = new CountDownLatch(1); + executor.execute(new Task(between(1, 500), doneLatch)); + safeAwait(doneLatch, TimeValue.ONE_MINUTE); + } + } finally { + ThreadPool.terminate(executor, 1, TimeUnit.SECONDS); + } + } } diff --git a/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java index 25ac11b516dc0..834f53dc410da 100644 --- a/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java @@ -34,20 +34,20 @@ public class EnvironmentTests extends ESTestCase { public void testRepositoryResolution() throws IOException { Environment environment = newEnvironment(); - assertThat(environment.resolveRepoFile("/test/repos/repo1"), nullValue()); - assertThat(environment.resolveRepoFile("test/repos/repo1"), nullValue()); + assertThat(environment.resolveRepoDir("/test/repos/repo1"), nullValue()); + assertThat(environment.resolveRepoDir("test/repos/repo1"), nullValue()); environment = newEnvironment( Settings.builder() .putList(Environment.PATH_REPO_SETTING.getKey(), "/test/repos", "/another/repos", "/test/repos/../other") .build() ); - assertThat(environment.resolveRepoFile("/test/repos/repo1"), notNullValue()); - assertThat(environment.resolveRepoFile("test/repos/repo1"), notNullValue()); - assertThat(environment.resolveRepoFile("/another/repos/repo1"), notNullValue()); - assertThat(environment.resolveRepoFile("/test/repos/../repo1"), nullValue()); - assertThat(environment.resolveRepoFile("/test/repos/../repos/repo1"), notNullValue()); - assertThat(environment.resolveRepoFile("/somethingeles/repos/repo1"), nullValue()); - assertThat(environment.resolveRepoFile("/test/other/repo"), notNullValue()); + assertThat(environment.resolveRepoDir("/test/repos/repo1"), notNullValue()); + assertThat(environment.resolveRepoDir("test/repos/repo1"), notNullValue()); + assertThat(environment.resolveRepoDir("/another/repos/repo1"), notNullValue()); + assertThat(environment.resolveRepoDir("/test/repos/../repo1"), nullValue()); + assertThat(environment.resolveRepoDir("/test/repos/../repos/repo1"), notNullValue()); + assertThat(environment.resolveRepoDir("/somethingeles/repos/repo1"), nullValue()); + assertThat(environment.resolveRepoDir("/test/other/repo"), notNullValue()); assertThat(environment.resolveRepoURL(new URL("file:///test/repos/repo1")), notNullValue()); assertThat(environment.resolveRepoURL(new URL("file:/test/repos/repo1")), notNullValue()); @@ -66,7 +66,7 @@ public void testPathDataWhenNotSet() { final Path pathHome = createTempDir().toAbsolutePath(); final Settings settings = Settings.builder().put("path.home", pathHome).build(); final Environment environment = new Environment(settings, null); - assertThat(environment.dataFiles(), equalTo(new Path[] { pathHome.resolve("data") })); + assertThat(environment.dataDirs(), equalTo(new Path[] { pathHome.resolve("data") })); } public void testPathDataNotSetInEnvironmentIfNotSet() { @@ -82,41 +82,41 @@ public void testPathDataLegacyCommaList() { .put("path.data", createTempDir().toAbsolutePath() + "," + createTempDir().toAbsolutePath()) .build(); final Environment environment = new Environment(settings, null); - assertThat(environment.dataFiles(), arrayWithSize(2)); + assertThat(environment.dataDirs(), arrayWithSize(2)); } public void testPathLogsWhenNotSet() { final Path pathHome = createTempDir().toAbsolutePath(); final Settings settings = Settings.builder().put("path.home", pathHome).build(); final Environment environment = new Environment(settings, null); - assertThat(environment.logsFile(), equalTo(pathHome.resolve("logs"))); + assertThat(environment.logsDir(), equalTo(pathHome.resolve("logs"))); } public void testDefaultConfigPath() { final Path path = createTempDir().toAbsolutePath(); final Settings settings = Settings.builder().put("path.home", path).build(); final Environment environment = new Environment(settings, null); - assertThat(environment.configFile(), equalTo(path.resolve("config"))); + assertThat(environment.configDir(), equalTo(path.resolve("config"))); } public void testConfigPath() { final Path configPath = createTempDir().toAbsolutePath(); final Settings settings = Settings.builder().put("path.home", createTempDir().toAbsolutePath()).build(); final Environment environment = new Environment(settings, configPath); - assertThat(environment.configFile(), equalTo(configPath)); + assertThat(environment.configDir(), equalTo(configPath)); } public void testConfigPathWhenNotSet() { final Path pathHome = createTempDir().toAbsolutePath(); final Settings settings = Settings.builder().put("path.home", pathHome).build(); final Environment environment = new Environment(settings, null); - assertThat(environment.configFile(), equalTo(pathHome.resolve("config"))); + assertThat(environment.configDir(), equalTo(pathHome.resolve("config"))); } public void testNonExistentTempPathValidation() { Settings build = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); Environment environment = new Environment(build, null, createTempDir().resolve("this_does_not_exist")); - FileNotFoundException e = expectThrows(FileNotFoundException.class, environment::validateTmpFile); + FileNotFoundException e = expectThrows(FileNotFoundException.class, environment::validateTmpDir); assertThat(e.getMessage(), startsWith("Temporary directory [")); assertThat(e.getMessage(), endsWith("this_does_not_exist] does not exist or is not accessible")); } @@ -124,7 +124,7 @@ public void testNonExistentTempPathValidation() { public void testTempPathValidationWhenRegularFile() throws IOException { Settings build = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); Environment environment = new Environment(build, null, createTempFile("something", ".test")); - IOException e = expectThrows(IOException.class, environment::validateTmpFile); + IOException e = expectThrows(IOException.class, environment::validateTmpDir); assertThat(e.getMessage(), startsWith("Temporary directory [")); assertThat(e.getMessage(), endsWith(".test] is not a directory")); } diff --git a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java index ab0ccb129fe57..0cfa9716c5fe7 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java @@ -131,7 +131,7 @@ public void testCleanupAll() throws Exception { boolean hasClusterState = randomBoolean(); createIndexDataFiles(dataMasterSettings, shardCount, hasClusterState); - String messageText = NodeRepurposeCommand.noMasterMessage(1, environment.dataFiles().length * shardCount, 0); + String messageText = NodeRepurposeCommand.noMasterMessage(1, environment.dataDirs().length * shardCount, 0); Matcher outputMatcher = allOf( containsString(messageText), @@ -157,7 +157,7 @@ public void testCleanupShardData() throws Exception { createIndexDataFiles(dataMasterSettings, shardCount, hasClusterState); Matcher matcher = allOf( - containsString(NodeRepurposeCommand.shardMessage(environment.dataFiles().length * shardCount, 1)), + containsString(NodeRepurposeCommand.shardMessage(environment.dataDirs().length * shardCount, 1)), conditionalNot(containsString("testUUID"), verbose == false), conditionalNot(containsString("testIndex"), verbose == false || hasClusterState == false), conditionalNot(containsString("no name for uuid: testUUID"), verbose == false || hasClusterState) @@ -271,7 +271,7 @@ private void verifyUnchangedDataFiles(CheckedRunnable runna private long digestPaths() { // use a commutative digest to avoid dependency on file system order. - return Arrays.stream(environment.dataFiles()).mapToLong(this::digestPath).sum(); + return Arrays.stream(environment.dataDirs()).mapToLong(this::digestPath).sum(); } private long digestPath(Path path) { diff --git a/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java b/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java index 521c045e32597..e882c259e7b87 100644 --- a/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java @@ -56,12 +56,12 @@ public void testCorsConfigWithBadRegex() { public void testCorsConfig() { final Set methods = new HashSet<>(Arrays.asList("get", "options", "post")); final Set headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length")); - final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements + final String maybeSpace = randomFrom(" ", ""); // sometimes have a leading whitespace between comma delimited elements final Settings settings = Settings.builder() .put(SETTING_CORS_ENABLED.getKey(), true) .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*") - .put(SETTING_CORS_ALLOW_METHODS.getKey(), collectionToDelimitedString(methods, ",", prefix, "")) - .put(SETTING_CORS_ALLOW_HEADERS.getKey(), collectionToDelimitedString(headers, ",", prefix, "")) + .put(SETTING_CORS_ALLOW_METHODS.getKey(), maybeSpace + collectionToDelimitedString(methods, "," + maybeSpace)) + .put(SETTING_CORS_ALLOW_HEADERS.getKey(), maybeSpace + collectionToDelimitedString(headers, "," + maybeSpace)) .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true) .build(); final CorsHandler.Config corsConfig = CorsHandler.buildConfig(settings); diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java index d43a1e09d71a3..05ebb19567b9f 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java @@ -59,6 +59,7 @@ public void testSortedDocValuesSingleUniqueValue() throws IOException { try (Directory directory = newDirectory()) { Analyzer analyzer = new MockAnalyzer(random()); IndexWriterConfig conf = newIndexWriterConfig(analyzer); + conf.setCodec(getCodec()); conf.setMergePolicy(newLogMergePolicy()); try (RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, conf)) { for (int i = 0; i < NUM_DOCS; i++) { @@ -95,6 +96,7 @@ public void testSortedSetDocValuesSingleUniqueValue() throws IOException { try (Directory directory = newDirectory()) { Analyzer analyzer = new MockAnalyzer(random()); IndexWriterConfig conf = newIndexWriterConfig(analyzer); + conf.setCodec(getCodec()); conf.setMergePolicy(newLogMergePolicy()); try (RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, conf)) { for (int i = 0; i < NUM_DOCS; i++) { @@ -134,6 +136,7 @@ public void testSortedSetDocValuesSingleUniqueValue() throws IOException { public void testOneDocManyValues() throws Exception { IndexWriterConfig config = new IndexWriterConfig(); + config.setCodec(getCodec()); try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, config)) { int numValues = 128 + random().nextInt(1024); // > 2^7 to require two blocks Document d = new Document(); @@ -161,6 +164,7 @@ public void testManyDocsWithManyValues() throws Exception { final Map sortedNumbers = new HashMap<>(); // key -> numbers try (Directory directory = newDirectory()) { IndexWriterConfig conf = newIndexWriterConfig(); + conf.setCodec(getCodec()); try (RandomIndexWriter writer = new RandomIndexWriter(random(), directory, conf)) { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java deleted file mode 100644 index 130c10130c4f3..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.Orientation; -import org.elasticsearch.core.Strings; -import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geo.ShapeTestUtils; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; -import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField; -import org.elasticsearch.lucene.spatial.CartesianShapeIndexer; -import org.elasticsearch.lucene.spatial.CoordinateEncoder; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.hamcrest.RectangleMatcher; -import org.elasticsearch.test.hamcrest.WellKnownBinaryBytesRefMatcher; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Optional; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.IntStream; - -public class AbstractShapeGeometryFieldMapperTests extends ESTestCase { - public void testCartesianBoundsBlockLoader() throws IOException { - testBoundsBlockLoaderAux( - CoordinateEncoder.CARTESIAN, - () -> ShapeTestUtils.randomGeometryWithoutCircle(0, false), - CartesianShapeIndexer::new, - SpatialEnvelopeVisitor::visitCartesian - ); - } - - // TODO when we turn this optimization on for geo, this test should pass. - public void ignoreTestGeoBoundsBlockLoader() throws IOException { - testBoundsBlockLoaderAux( - CoordinateEncoder.GEO, - () -> GeometryTestUtils.randomGeometryWithoutCircle(0, false), - field -> new GeoShapeIndexer(Orientation.RIGHT, field), - g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP) - ); - } - - private static void testBoundsBlockLoaderAux( - CoordinateEncoder encoder, - Supplier generator, - Function indexerFactory, - Function> visitor - ) throws IOException { - var geometries = IntStream.range(0, 50).mapToObj(i -> generator.get()).toList(); - var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field", encoder); - try (Directory directory = newDirectory()) { - try (var iw = new RandomIndexWriter(random(), directory)) { - for (Geometry geometry : geometries) { - var shape = new BinaryShapeDocValuesField("field", encoder); - shape.add(indexerFactory.apply("field").indexShape(geometry), geometry); - var doc = new Document(); - doc.add(shape); - iw.addDocument(doc); - } - } - - var expected = new ArrayList(); - var byteRefResults = new ArrayList(); - int currentIndex = 0; - try (DirectoryReader reader = DirectoryReader.open(directory)) { - for (var leaf : reader.leaves()) { - LeafReader leafReader = leaf.reader(); - int numDocs = leafReader.numDocs(); - // We specifically check just the even indices, to verify the loader can skip documents correctly. - int[] array = evenArray(numDocs); - for (int i = 0; i < array.length; i += 1) { - expected.add(visitor.apply(geometries.get(array[i] + currentIndex)).get()); - } - try (var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array))) { - for (int i = 0; i < block.size(); i++) { - byteRefResults.add((BytesRef) block.get(i)); - } - } - currentIndex += numDocs; - } - } - - for (int i = 0; i < expected.size(); i++) { - Rectangle rectangle = expected.get(i); - var geoString = rectangle.toString(); - assertThat( - Strings.format("geometry '%s' wasn't extracted correctly", geoString), - byteRefResults.get(i), - WellKnownBinaryBytesRefMatcher.encodes(RectangleMatcher.closeToFloat(rectangle, 1e-3, encoder)) - ); - } - } - } - - private static int[] evenArray(int maxIndex) { - return IntStream.range(0, maxIndex / 2).map(x -> x * 2).toArray(); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index fcadc7b238a43..553018b025e23 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -152,7 +152,8 @@ protected List exampleMalformedValues() { return List.of( exampleMalformedValue("2016-03-99").mapping(mappingWithFormat("strict_date_optional_time||epoch_millis")) .errorMatches("failed to parse date field [2016-03-99] with format [strict_date_optional_time||epoch_millis]"), - exampleMalformedValue("-522000000").mapping(mappingWithFormat("date_optional_time")).errorMatches("long overflow"), + exampleMalformedValue("-522000000").mapping(mappingWithFormat("date_optional_time")).errorMatches("too far in the past"), + exampleMalformedValue("522000000").mapping(mappingWithFormat("date_optional_time")).errorMatches("too far in the future"), exampleMalformedValue("2020").mapping(mappingWithFormat("strict_date")) .errorMatches("failed to parse date field [2020] with format [strict_date]"), exampleMalformedValue("hello world").mapping(mappingWithFormat("strict_date_optional_time")) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index d12bf5dc2e34c..628b64de19bd1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -2420,6 +2420,34 @@ public void testStoredArrayWithFlatFields() throws IOException { {"outer":{"inner":[{"a.b":"a.b","a.c":"a.c"}]}}""", syntheticSource); } + public void testSingleDeepIgnoredField() throws IOException { + DocumentMapper documentMapper = createSytheticSourceMapperService(mapping(b -> { + b.startObject("top"); + b.startObject("properties"); + { + b.startObject("level1").startObject("properties"); + { + b.startObject("level2").startObject("properties"); + { + b.startObject("n") + .field("type", "integer") + .field("doc_values", "false") + .field("synthetic_source_keep", "all") + .endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("top").startObject("level1").startObject("level2").field("n", 25).endObject().endObject().endObject(); + }); + assertEquals("{\"top\":{\"level1\":{\"level2\":{\"n\":25}}}}", syntheticSource); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 507314b31d00d..c89753214b6a9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -15,6 +15,9 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -26,6 +29,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -304,16 +308,56 @@ public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { public void testIsMetadataField() throws IOException { IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); - MapperService mapperService = createMapperService(settings, mapping(b -> {})); - assertFalse(mapperService.isMetadataField(randomAlphaOfLengthBetween(10, 15))); + CheckedFunction initMapperService = (indexMode) -> { + Settings.Builder settingsBuilder = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, version) + .put(IndexSettings.MODE.getKey(), indexMode); - for (String builtIn : IndicesModule.getBuiltInMetadataFields()) { - if (NestedPathFieldMapper.NAME.equals(builtIn) && version.before(IndexVersions.V_8_0_0)) { - continue; // Nested field does not exist in the 7x line + if (indexMode == IndexMode.TIME_SERIES) { + settingsBuilder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } - assertTrue("Expected " + builtIn + " to be a metadata field for version " + version, mapperService.isMetadataField(builtIn)); + + return createMapperService(settingsBuilder.build(), mapping(b -> {})); + }; + + Consumer assertMapperService = (mapperService) -> { + assertFalse(mapperService.isMetadataField(randomAlphaOfLengthBetween(10, 15))); + + for (String builtIn : IndicesModule.getBuiltInMetadataFields()) { + if (NestedPathFieldMapper.NAME.equals(builtIn) && version.before(IndexVersions.V_8_0_0)) { + continue; // Nested field does not exist in the 7x line + } + boolean isTimeSeriesField = builtIn.equals("_tsid") || builtIn.equals("_ts_routing_hash"); + boolean isTimeSeriesMode = mapperService.getIndexSettings().getMode().equals(IndexMode.TIME_SERIES); + + if (isTimeSeriesField && isTimeSeriesMode == false) { + assertFalse( + "Expected " + + builtIn + + " to not be a metadata field for version " + + version + + " and index mode " + + mapperService.getIndexSettings().getMode(), + mapperService.isMetadataField(builtIn) + ); + } else { + assertTrue( + "Expected " + + builtIn + + " to be a metadata field for version " + + version + + " and index mode " + + mapperService.getIndexSettings().getMode(), + mapperService.isMetadataField(builtIn) + ); + } + } + }; + + for (IndexMode indexMode : IndexMode.values()) { + MapperService mapperService = initMapperService.apply(indexMode); + assertMapperService.accept(mapperService); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java new file mode 100644 index 0000000000000..0322286277b25 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShapeGeometryFieldMapperTests.java @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Document; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.geo.GeometryNormalizer; +import org.elasticsearch.core.Strings; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField; +import org.elasticsearch.lucene.spatial.CartesianShapeIndexer; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.hamcrest.RectangleMatcher; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.elasticsearch.common.geo.Orientation.RIGHT; + +public class ShapeGeometryFieldMapperTests extends ESTestCase { + public void testCartesianBoundsBlockLoader() throws IOException { + testBoundsBlockLoader( + CoordinateEncoder.CARTESIAN, + () -> ShapeTestUtils.randomGeometryWithoutCircle(0, false), + CartesianShapeIndexer::new, + SpatialEnvelopeVisitor::visitCartesian, + ShapeGeometryFieldMapperTests::makeCartesianRectangle + ); + } + + // TODO: Re-enable this test after fixing the bug in the ShapeEnvelopeVisitor regarding Rectangle crossing the dateline + // Currently it is flaky if the geometries include a Rectangle like one defined in the test below + public void ignoreTestGeoBoundsBlockLoader() throws IOException { + testBoundsBlockLoader( + CoordinateEncoder.GEO, + () -> normalize(GeometryTestUtils.randomGeometryWithoutCircle(0, false)), + field -> new GeoShapeIndexer(RIGHT, field), + g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP), + ShapeGeometryFieldMapperTests::makeGeoRectangle + ); + } + + // TODO: Re-enable this test after fixing the bug in the SpatialEnvelopeVisitor regarding Rectangle crossing the dateline + // See the difference between GeoShapeIndexer.visitRectangle() and SpatialEnvelopeVisitor.GeoPointVisitor.visitRectangle() + public void ignoreTestRectangleCrossingDateline() throws IOException { + var geometries = new ArrayList(); + geometries.add(new Rectangle(180, 51.62247094594227, -18.5, -24.902304006345503)); + testBoundsBlockLoaderAux( + CoordinateEncoder.GEO, + geometries, + field -> new GeoShapeIndexer(RIGHT, field), + g -> SpatialEnvelopeVisitor.visitGeo(g, SpatialEnvelopeVisitor.WrapLongitude.WRAP), + ShapeGeometryFieldMapperTests::makeGeoRectangle + ); + } + + private Geometry normalize(Geometry geometry) { + return GeometryNormalizer.needsNormalize(RIGHT, geometry) ? GeometryNormalizer.apply(RIGHT, geometry) : geometry; + } + + private static void testBoundsBlockLoader( + CoordinateEncoder encoder, + Supplier generator, + Function indexerFactory, + Function> visitor, + BiFunction rectangleMaker + ) throws IOException { + var geometries = IntStream.range(0, 50).mapToObj(i -> generator.get()).toList(); + testBoundsBlockLoaderAux(encoder, geometries, indexerFactory, visitor, rectangleMaker); + } + + private static void testBoundsBlockLoaderAux( + CoordinateEncoder encoder, + java.util.List geometries, + Function indexerFactory, + Function> visitor, + BiFunction rectangleMaker + ) throws IOException { + var loader = new AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader("field"); + try (Directory directory = newDirectory()) { + try (var iw = new RandomIndexWriter(random(), directory)) { + for (Geometry geometry : geometries) { + var shape = new BinaryShapeDocValuesField("field", encoder); + shape.add(indexerFactory.apply("field").indexShape(geometry), geometry); + var doc = new Document(); + doc.add(shape); + iw.addDocument(doc); + } + } + + var expected = new ArrayList(); + ArrayList intArrayResults = new ArrayList<>(); + int currentIndex = 0; + try (DirectoryReader reader = DirectoryReader.open(directory)) { + for (var leaf : reader.leaves()) { + LeafReader leafReader = leaf.reader(); + int numDocs = leafReader.numDocs(); + // We specifically check just the even indices, to verify the loader can skip documents correctly. + int[] array = evenArray(numDocs); + for (int j : array) { + expected.add(visitor.apply(geometries.get(j + currentIndex)).get()); + } + try (var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array))) { + for (int i = 0; i < block.size(); i++) { + intArrayResults.add(block.get(i)); + } + } + currentIndex += numDocs; + } + } + + for (int i = 0; i < expected.size(); i++) { + Rectangle rectangle = expected.get(i); + var geoString = rectangle.toString(); + Rectangle result = rectangleMaker.apply(encoder, intArrayResults.get(i)); + assertThat( + Strings.format("geometry[%d] '%s' wasn't extracted correctly", i, geoString), + result, + RectangleMatcher.closeToFloat(rectangle, 1e-3, encoder) + ); + } + } + } + + private static Rectangle makeCartesianRectangle(CoordinateEncoder encoder, Object integers) { + if (integers instanceof ArrayList list) { + int[] ints = list.stream().mapToInt(x -> (int) x).toArray(); + if (list.size() == 6) { + // Data in order defined by Extent class + double top = encoder.decodeY(ints[0]); + double bottom = encoder.decodeY(ints[1]); + double negLeft = encoder.decodeX(ints[2]); + double negRight = encoder.decodeX(ints[3]); + double posLeft = encoder.decodeX(ints[4]); + double posRight = encoder.decodeX(ints[5]); + return new Rectangle(Math.min(negLeft, posLeft), Math.max(negRight, posRight), top, bottom); + } else if (list.size() == 4) { + // Data in order defined by Rectangle class + return new Rectangle( + encoder.decodeX(ints[0]), + encoder.decodeX(ints[1]), + encoder.decodeY(ints[2]), + encoder.decodeY(ints[3]) + ); + } else { + throw new IllegalArgumentException("Expected 4 or 6 integers"); + } + } + throw new IllegalArgumentException("Expected an array of integers"); + } + + private static Rectangle makeGeoRectangle(CoordinateEncoder encoder, Object integers) { + if (integers instanceof ArrayList list) { + int[] ints = list.stream().mapToInt(x -> (int) x).toArray(); + if (list.size() != 6) { + throw new IllegalArgumentException("Expected 6 integers"); + } + // Data in order defined by Extent class + return asGeoRectangle(ints[0], ints[1], ints[2], ints[3], ints[4], ints[5]); + } + throw new IllegalArgumentException("Expected an array of integers"); + } + + private static Rectangle asGeoRectangle(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + return SpatialEnvelopeVisitor.GeoPointVisitor.getResult( + GeoEncodingUtils.decodeLatitude(top), + GeoEncodingUtils.decodeLatitude(bottom), + negLeft <= 0 ? decodeLongitude(negLeft) : Double.POSITIVE_INFINITY, + negRight <= 0 ? decodeLongitude(negRight) : Double.NEGATIVE_INFINITY, + posLeft >= 0 ? decodeLongitude(posLeft) : Double.POSITIVE_INFINITY, + posRight >= 0 ? decodeLongitude(posRight) : Double.NEGATIVE_INFINITY, + SpatialEnvelopeVisitor.WrapLongitude.WRAP + ); + } + + private static int[] evenArray(int maxIndex) { + return IntStream.range(0, maxIndex / 2).map(x -> x * 2).toArray(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index bc560d94b8f52..8441d375777ad 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -253,14 +253,14 @@ public void testSyntheticSourceInTimeSeries() throws IOException { }); DocumentMapper mapper = createTimeSeriesModeDocumentMapper(mapping); assertTrue(mapper.sourceMapper().isSynthetic()); - assertEquals("{\"_source\":{}}", mapper.sourceMapper().toString()); + assertEquals("{}", mapper.sourceMapper().toString()); } public void testSyntheticSourceWithLogsIndexMode() throws IOException { XContentBuilder mapping = fieldMapping(b -> { b.field("type", "keyword"); }); DocumentMapper mapper = createLogsModeDocumentMapper(mapping); assertTrue(mapper.sourceMapper().isSynthetic()); - assertEquals("{\"_source\":{}}", mapper.sourceMapper().toString()); + assertEquals("{}", mapper.sourceMapper().toString()); } public void testSupportsNonDefaultParameterValues() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java new file mode 100644 index 0000000000000..dcd6c35ab1700 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class StringStoredFieldFieldLoaderTests extends ESTestCase { + + public void testLoadStoredFieldAndReset() throws IOException { + var sfl = new StringStoredFieldFieldLoader("foo", "foo") { + @Override + protected void write(XContentBuilder b, Object value) throws IOException { + b.value((String) value); + } + }; + + var storedFieldLoaders = sfl.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + storedFieldLoaders.get("foo").load(List.of("one")); + + var result = XContentBuilder.builder(XContentType.JSON.xContent()); + result.startObject(); + sfl.write(result); + result.endObject(); + + assertEquals(""" + {"foo":"one"}""", Strings.toString(result)); + + var empty = XContentBuilder.builder(XContentType.JSON.xContent()); + empty.startObject(); + // reset() should have been called after previous write + sfl.write(empty); + empty.endObject(); + + assertEquals("{}", Strings.toString(empty)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java index 92afe312e5d13..bc3b44b908246 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java @@ -45,7 +45,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.security.AccessControlException; import java.util.Arrays; import java.util.Map; import java.util.function.Predicate; @@ -259,7 +258,7 @@ public boolean hardLinksSupported(Path path) throws IOException { BasicFileAttributes sourceAttr = Files.readAttributes(path.resolve("foo.bar"), BasicFileAttributes.class); // we won't get here - no permission ;) return destAttr.fileKey() != null && destAttr.fileKey().equals(sourceAttr.fileKey()); - } catch (AccessControlException ex) { + } catch (SecurityException ex) { return true; // if we run into that situation we know it's supported. } catch (UnsupportedOperationException ex) { return false; diff --git a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java index 394ce35c6b493..46e0d0d31000e 100644 --- a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java @@ -9,9 +9,12 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NoLockFactory; @@ -67,6 +70,29 @@ public void testPreload() throws IOException { } } + public void testDisableRandomAdvice() throws IOException { + Directory dir = new FilterDirectory(new ByteBuffersDirectory()) { + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + assertFalse(context.randomAccess); + return super.openInput(name, context); + } + }; + Directory noRandomAccessDir = FsDirectoryFactory.disableRandomAdvice(dir); + try (IndexOutput out = noRandomAccessDir.createOutput("foo", IOContext.DEFAULT)) { + out.writeInt(42); + } + // Test the tester + expectThrows(AssertionError.class, () -> dir.openInput("foo", IOContext.RANDOM)); + + // The wrapped directory shouldn't fail regardless of the IOContext + for (IOContext context : Arrays.asList(IOContext.READ, IOContext.DEFAULT, IOContext.READONCE, IOContext.RANDOM)) { + try (IndexInput in = noRandomAccessDir.openInput("foo", context)) { + assertEquals(42, in.readInt()); + } + } + } + private Directory newDirectory(Settings settings) throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(idxSettings.getUUID()).resolve("0"); diff --git a/server/src/test/java/org/elasticsearch/indices/ExecutorSelectorTests.java b/server/src/test/java/org/elasticsearch/indices/ExecutorSelectorTests.java index dacf3c28fce02..ff02a07b878b1 100644 --- a/server/src/test/java/org/elasticsearch/indices/ExecutorSelectorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/ExecutorSelectorTests.java @@ -84,6 +84,7 @@ public void testDefaultSystemDataStreamThreadPools() { .build(), Map.of(), Collections.singletonList("test"), + "test", null ) ) @@ -116,6 +117,7 @@ public void testCustomSystemDataStreamThreadPools() { .build(), Map.of(), Collections.singletonList("test"), + "test", new ExecutorNames( ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java index 77f3d75015d0d..eb59d5e99ecec 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java @@ -9,6 +9,8 @@ package org.elasticsearch.indices; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -291,4 +293,36 @@ public void testMappingsVersions() { assertThat(mappingsVersions.get(".managed-primary").version(), equalTo(3)); assertThat(mappingsVersions.keySet(), not(contains("unmanaged"))); } + + public void testSystemDataStreamPattern() { + String dataStreamName = ".my-data-stream"; + SystemDataStreamDescriptor dataStreamDescriptor = new SystemDataStreamDescriptor( + dataStreamName, + "", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder().build(), + Map.of(), + Collections.singletonList("origin"), + "origin", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ); + + final SystemIndices systemIndices = new SystemIndices( + List.of( + new SystemIndices.Feature("test", "test feature", Collections.emptyList(), Collections.singletonList(dataStreamDescriptor)) + ) + ); + assertThat( + systemIndices.isSystemIndexBackingDataStream(DataStream.BACKING_INDEX_PREFIX + dataStreamName + "-2025.03.07-000001"), + equalTo(true) + ); + assertThat( + systemIndices.isSystemIndexBackingDataStream(DataStream.FAILURE_STORE_PREFIX + dataStreamName + "-2025.03.07-000001"), + equalTo(true) + ); + assertThat(systemIndices.isSystemIndexBackingDataStream(".migrated-ds-" + dataStreamName + "-2025.03.07-000001"), equalTo(true)); + assertThat(systemIndices.isSystemIndexBackingDataStream(".migrated-" + dataStreamName + "-2025.03.07-000001"), equalTo(false)); + assertThat(systemIndices.isSystemIndexBackingDataStream(dataStreamName), equalTo(false)); + assertThat(systemIndices.isSystemIndexBackingDataStream(dataStreamName + "-2025.03.07-000001"), equalTo(false)); + } } diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 2c8c648640eaf..9312725350f4b 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -454,7 +454,7 @@ public void testRegisterHunspellDictionary() throws Exception { InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff"); InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic"); Dictionary dictionary; - try (Directory tmp = newFSDirectory(environment.tmpFile())) { + try (Directory tmp = newFSDirectory(environment.tmpDir())) { dictionary = new Dictionary(tmp, "hunspell", aff, dic); } AnalysisModule module = new AnalysisModule(environment, singletonList(new AnalysisPlugin() { diff --git a/server/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java b/server/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java index 1a27954eed98b..8530fd21ea77d 100644 --- a/server/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java @@ -64,7 +64,7 @@ public void testDicWithNoAff() throws Exception { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { final Environment environment = new Environment(settings, getDataPath("/indices/analyze/no_aff_conf_dir")); new HunspellService(settings, environment, emptyMap()).getDictionary("en_US"); }); @@ -78,7 +78,7 @@ public void testDicWithTwoAffs() throws Exception { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .build(); - IllegalStateException e = expectThrows(IllegalStateException.class, () -> { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { final Environment environment = new Environment(settings, getDataPath("/indices/analyze/two_aff_conf_dir")); new HunspellService(settings, environment, emptyMap()).getDictionary("en_US"); }); diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index 0e8c7e0857251..3022e1670cb6a 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -279,6 +279,7 @@ public void testNoScriptCompilation() { propertyValue, scriptService ); + assertThat(result, instanceOf(ConfigurationUtils.ConstantTemplateScriptFactory.class)); assertThat(result.newInstance(null).execute(), equalTo(propertyValue)); verify(scriptService, times(0)).compile(any(), any()); } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 0e3e6181c5133..37dee399efde8 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2146,7 +2146,7 @@ public void testStatName() { Processor processor = mock(Processor.class); String name = randomAlphaOfLength(10); when(processor.getType()).thenReturn(name); - assertThat(IngestService.getProcessorName(processor), equalTo(name)); + assertThat(IngestService.getProcessorName(processor), sameInstance(name)); String tag = randomAlphaOfLength(10); when(processor.getTag()).thenReturn(tag); assertThat(IngestService.getProcessorName(processor), equalTo(name + ":" + tag)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java index dc3fb2a473f43..6586f180db875 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java @@ -19,6 +19,9 @@ import java.util.Map; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; public class IngestStatsTests extends ESTestCase { @@ -31,6 +34,71 @@ public void testSerialization() throws IOException { assertIngestStats(ingestStats, serializedStats); } + public void testIdentitySerialization() throws IOException { + IngestStats serializedStats = serialize(IngestStats.IDENTITY); + assertThat(serializedStats, sameInstance(IngestStats.IDENTITY)); + } + + public void testProcessorNameAndTypeIdentitySerialization() throws IOException { + IngestStats.Builder builder = new IngestStats.Builder(); + builder.addPipelineMetrics("pipeline_id", new IngestPipelineMetric()); + builder.addProcessorMetrics("pipeline_id", "set", "set", new IngestMetric()); + builder.addProcessorMetrics("pipeline_id", "set:foo", "set", new IngestMetric()); + builder.addProcessorMetrics("pipeline_id", "set:bar", "set", new IngestMetric()); + builder.addTotalMetrics(new IngestMetric()); + + IngestStats serializedStats = serialize(builder.build()); + List processorStats = serializedStats.processorStats().get("pipeline_id"); + + // these are just table stakes + assertThat(processorStats.get(0).name(), is("set")); + assertThat(processorStats.get(0).type(), is("set")); + assertThat(processorStats.get(1).name(), is("set:foo")); + assertThat(processorStats.get(1).type(), is("set")); + assertThat(processorStats.get(2).name(), is("set:bar")); + assertThat(processorStats.get(2).type(), is("set")); + + // this is actually interesting, though -- we're canonical-izing these strings to keep our heap usage under control + final String set = processorStats.get(0).name(); + assertThat(processorStats.get(0).name(), sameInstance(set)); + assertThat(processorStats.get(0).type(), sameInstance(set)); + assertThat(processorStats.get(1).type(), sameInstance(set)); + assertThat(processorStats.get(2).type(), sameInstance(set)); + } + + public void testBytesStatsSerialization() throws IOException { + { + IngestPipelineMetric metric = new IngestPipelineMetric(); + IngestStats.ByteStats byteStats = metric.createByteStats(); + assertThat(byteStats, sameInstance(IngestStats.ByteStats.IDENTITY)); + + IngestStats.ByteStats serializedByteStats = serialize(byteStats); + assertThat(serializedByteStats, sameInstance(IngestStats.ByteStats.IDENTITY)); + assertThat(IngestStats.ByteStats.merge(IngestStats.ByteStats.IDENTITY, byteStats), sameInstance(byteStats)); + } + { + long ingestBytes = randomLongBetween(0, Long.MAX_VALUE); + long producedBytes = randomLongBetween(0, Long.MAX_VALUE); + IngestPipelineMetric metric = new IngestPipelineMetric(); + metric.preIngestBytes(ingestBytes); + metric.postIngestBytes(producedBytes); + IngestStats.ByteStats byteStats = metric.createByteStats(); + assertThat(byteStats.bytesIngested(), equalTo(ingestBytes)); + assertThat(byteStats.bytesProduced(), equalTo(producedBytes)); + + IngestStats.ByteStats serializedByteStats = serialize(byteStats); + assertThat(serializedByteStats.bytesIngested(), equalTo(ingestBytes)); + assertThat(serializedByteStats.bytesProduced(), equalTo(producedBytes)); + + assertThat(IngestStats.ByteStats.merge(byteStats, IngestStats.ByteStats.IDENTITY), sameInstance(byteStats)); + assertThat(IngestStats.ByteStats.merge(IngestStats.ByteStats.IDENTITY, byteStats), sameInstance(byteStats)); + assertThat( + IngestStats.ByteStats.merge(IngestStats.ByteStats.IDENTITY, IngestStats.ByteStats.IDENTITY), + sameInstance(IngestStats.ByteStats.IDENTITY) + ); + } + } + public void testStatsMerge() { var first = randomStats(); var second = randomStats(); @@ -239,6 +307,13 @@ private static IngestStats serialize(IngestStats stats) throws IOException { return IngestStats.read(in); } + private static IngestStats.ByteStats serialize(IngestStats.ByteStats stats) throws IOException { + var out = new BytesStreamOutput(); + stats.writeTo(out); + var in = out.bytes().streamInput(); + return IngestStats.readByteStats(in); + } + private static void assertIngestStats(IngestStats ingestStats, IngestStats serializedStats) { assertNotSame(ingestStats, serializedStats); assertNotSame(ingestStats.totalStats(), serializedStats.totalStats()); diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java index 7be6e97762ccf..78e3213e690a8 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -28,9 +28,11 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -143,6 +145,41 @@ public void testGetVersion() { } } + @SuppressWarnings("unchecked") + public void testMapKeyOrderingRoundTrip() throws IOException { + // make up two random keys + String key1 = randomAlphaOfLength(10); + String key2 = randomValueOtherThan(key1, () -> randomAlphaOfLength(10)); + // stick them as mappings onto themselves in the _meta of a pipeline configuration + // this happens to use the _meta as a convenient map to test that the ordering of the key sets is the same + String configJson = Strings.format(""" + {"description": "blah", "_meta" : {"foo": "bar", "%s": "%s", "%s": "%s"}}""", key1, key1, key2, key2); + PipelineConfiguration configuration = new PipelineConfiguration( + "1", + new BytesArray(configJson.getBytes(StandardCharsets.UTF_8)), + XContentType.JSON + ); + + // serialize it to bytes + XContentType xContentType = randomFrom(XContentType.values()); + final BytesReference bytes; + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + configuration.toXContent(builder, ToXContent.EMPTY_PARAMS); + bytes = BytesReference.bytes(builder); + } + + // deserialize it back + ContextParser parser = PipelineConfiguration.getParser(); + XContentParser xContentParser = xContentType.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput()); + PipelineConfiguration parsed = parser.parse(xContentParser, null); + + // make sure the _meta key sets are in the same order + Set keys1 = ((Map) configuration.getConfig().get("_meta")).keySet(); + Set keys2 = ((Map) parsed.getConfig().get("_meta")).keySet(); + assertThat(keys1, contains(keys2.toArray(new String[0]))); + } + @Override protected PipelineConfiguration createTestInstance() { BytesArray config; diff --git a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java index f5266568e6fdf..56a9e1629d51e 100644 --- a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java @@ -157,7 +157,7 @@ private void assertHighlightOneDoc( noMatchSize, expectedPassages.length, maxAnalyzedOffset, - queryMaxAnalyzedOffset, + QueryMaxAnalyzedOffset.create(queryMaxAnalyzedOffset, maxAnalyzedOffset), true, true ); diff --git a/server/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java b/server/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java index 32edcc0ad82aa..c0e1c1143ef42 100644 --- a/server/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java +++ b/server/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java @@ -57,7 +57,7 @@ public void testEmptySettings() { assertEquals(defaultNodeName, settings.get("node.name")); assertNotNull(settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())); // a cluster name was set String home = Environment.PATH_HOME_SETTING.get(baseEnvSettings); - String configDir = env.configFile().toString(); + String configDir = env.configDir().toString(); assertTrue(configDir, configDir.startsWith(home)); assertEquals("elasticsearch", settings.get("cluster.name")); } diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java index 97158e27b8528..8129f67947cf9 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java @@ -52,7 +52,7 @@ public class PluginsLoaderTests extends ESTestCase { static PluginsLoader newPluginsLoader(Settings settings) { return PluginsLoader.createPluginsLoader( Set.of(), - PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsFile()), + PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsDir()), Map.of(), false ); @@ -121,7 +121,7 @@ public void testStablePluginWithNativeAccess() throws Exception { var pluginsLoader = PluginsLoader.createPluginsLoader( Set.of(), - PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsFile()), + PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsDir()), Map.of(STABLE_PLUGIN_NAME, Set.of(STABLE_PLUGIN_MODULE_NAME)), false ); @@ -182,7 +182,7 @@ public void testModularPluginLoadingWithNativeAccess() throws Exception { var pluginsLoader = PluginsLoader.createPluginsLoader( Set.of(), - PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsFile()), + PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsDir()), Map.of(MODULAR_PLUGIN_NAME, Set.of(MODULAR_PLUGIN_MODULE_NAME)), false ); diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index e76994f69c01e..ef12e767c1e28 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.plugin.analysis.CharFilterFactory; import org.elasticsearch.plugins.scanners.PluginInfo; import org.elasticsearch.plugins.spi.BarPlugin; @@ -70,7 +71,7 @@ static PluginsService newPluginsService(Settings settings) { null, PluginsLoader.createPluginsLoader( Set.of(), - PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsFile()), + PluginsLoader.loadPluginsBundles(TestEnvironment.newEnvironment(settings).pluginsDir()), Map.of(), false ) @@ -873,6 +874,7 @@ public Reader create(Reader reader) { } public void testCanCreateAClassLoader() { + assumeTrue("security manager must be available", RuntimeVersionFeature.isSecurityManagerAvailable()); assertEquals( "access denied (\"java.lang.RuntimePermission\" \"createClassLoader\")", expectThrows(AccessControlException.class, () -> new Loader(this.getClass().getClassLoader())).getMessage() diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java index f5ebacde08820..250d10855b23f 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java @@ -40,9 +40,12 @@ import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN; import static org.elasticsearch.repositories.RepositoryData.MISSING_UUID; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.not; /** * Tests for the {@link RepositoryData} class. @@ -430,6 +433,19 @@ public void testFailsIfMinVersionNotSatisfied() throws IOException { } } + public void testToString() { + final var repositoryData = generateRandomRepoData(); + assertThat( + repositoryData.toString(), + allOf( + containsString("RepositoryData"), + containsString(repositoryData.getUuid()), + containsString(Long.toString(repositoryData.getGenId())), + not(containsString("@")) // not the default Object#toString which does a very expensive hashcode computation + ) + ); + } + public static RepositoryData generateRandomRepoData() { final int numIndices = randomIntBetween(1, 30); final List indices = new ArrayList<>(numIndices); diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index 1c8568a9d1a92..2a7b8f3487596 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -113,7 +113,7 @@ public void setUp() throws Exception { clusterService.getMasterService().setClusterStateSupplier(() -> clusterState); env = newEnvironment(Settings.EMPTY); - Files.createDirectories(env.configFile()); + Files.createDirectories(env.configDir()); ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); @@ -159,7 +159,7 @@ public void testStartStop() { public void testOperatorDirName() { Path operatorPath = fileSettingsService.watchedFileDir(); - assertTrue(operatorPath.startsWith(env.configFile())); + assertTrue(operatorPath.startsWith(env.configDir())); assertTrue(operatorPath.endsWith("operator")); Path operatorSettingsFile = fileSettingsService.watchedFile(); diff --git a/server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java b/server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java index 74c6fceddf71b..c58621d03ce8f 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.http.HttpResponse; import org.elasticsearch.tasks.Task; @@ -44,6 +45,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; public class RestCancellableNodeClientTests extends ESTestCase { @@ -148,8 +150,42 @@ public void testChannelAlreadyClosed() { assertEquals(totalSearches, testClient.cancelledTasks.size()); } + public void testConcurrentExecuteAndClose() throws Exception { + final var testClient = new TestClient(Settings.EMPTY, threadPool, true); + int initialHttpChannels = RestCancellableNodeClient.getNumChannels(); + int numTasks = randomIntBetween(1, 30); + TestHttpChannel channel = new TestHttpChannel(); + final var startLatch = new CountDownLatch(1); + final var doneLatch = new CountDownLatch(numTasks + 1); + final var expectedTasks = Sets.newHashSetWithExpectedSize(numTasks); + for (int j = 0; j < numTasks; j++) { + RestCancellableNodeClient client = new RestCancellableNodeClient(testClient, channel); + threadPool.generic().execute(() -> { + client.execute(TransportSearchAction.TYPE, new SearchRequest(), ActionListener.running(ESTestCase::fail)); + startLatch.countDown(); + doneLatch.countDown(); + }); + expectedTasks.add(new TaskId(testClient.getLocalNodeId(), j)); + } + threadPool.generic().execute(() -> { + try { + safeAwait(startLatch); + channel.awaitClose(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } finally { + doneLatch.countDown(); + } + }); + safeAwait(doneLatch); + assertEquals(initialHttpChannels, RestCancellableNodeClient.getNumChannels()); + assertEquals(expectedTasks, testClient.cancelledTasks); + } + private static class TestClient extends NodeClient { - private final AtomicLong counter = new AtomicLong(0); + private final LongSupplier searchTaskIdGenerator = new AtomicLong(0)::getAndIncrement; + private final LongSupplier cancelTaskIdGenerator = new AtomicLong(1000)::getAndIncrement; private final Set cancelledTasks = new CopyOnWriteArraySet<>(); private final AtomicInteger searchRequests = new AtomicInteger(0); private final boolean timeout; @@ -167,9 +203,17 @@ public Task exe ) { switch (action.name()) { case TransportCancelTasksAction.NAME -> { - CancelTasksRequest cancelTasksRequest = (CancelTasksRequest) request; - assertTrue("tried to cancel the same task more than once", cancelledTasks.add(cancelTasksRequest.getTargetTaskId())); - Task task = request.createTask(counter.getAndIncrement(), "cancel_task", action.name(), null, Collections.emptyMap()); + assertTrue( + "tried to cancel the same task more than once", + cancelledTasks.add(asInstanceOf(CancelTasksRequest.class, request).getTargetTaskId()) + ); + Task task = request.createTask( + cancelTaskIdGenerator.getAsLong(), + "cancel_task", + action.name(), + null, + Collections.emptyMap() + ); if (randomBoolean()) { listener.onResponse(null); } else { @@ -180,7 +224,13 @@ public Task exe } case TransportSearchAction.NAME -> { searchRequests.incrementAndGet(); - Task searchTask = request.createTask(counter.getAndIncrement(), "search", action.name(), null, Collections.emptyMap()); + Task searchTask = request.createTask( + searchTaskIdGenerator.getAsLong(), + "search", + action.name(), + null, + Collections.emptyMap() + ); if (timeout == false) { if (rarely()) { // make sure that search is sometimes also called from the same thread before the task is returned @@ -191,7 +241,7 @@ public Task exe } return searchTask; } - default -> throw new UnsupportedOperationException(); + default -> throw new AssertionError("unexpected action " + action.name()); } } @@ -222,10 +272,7 @@ public InetSocketAddress getRemoteAddress() { @Override public void close() { - if (open.compareAndSet(true, false) == false) { - assert false : "HttpChannel is already closed"; - return; // nothing to do - } + assertTrue("HttpChannel is already closed", open.compareAndSet(true, false)); ActionListener listener = closeListener.get(); if (listener != null) { boolean failure = randomBoolean(); @@ -241,6 +288,7 @@ public void close() { } private void awaitClose() throws InterruptedException { + assertNotNull("must set closeListener before calling awaitClose", closeListener.get()); close(); closeLatch.await(); } @@ -257,7 +305,7 @@ public void addCloseListener(ActionListener listener) { listener.onResponse(null); } else { if (closeListener.compareAndSet(null, listener) == false) { - throw new IllegalStateException("close listener already set, only one is allowed!"); + throw new AssertionError("close listener already set, only one is allowed!"); } } } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index df81e8ebcbb16..b60afca0939ae 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -78,6 +78,37 @@ public void testXContentChunked() throws IOException { assertThat(Strings.toString(builder), equalTo(expected)); } + public void testXContentChunkedHistory() throws Exception { + ScriptStats stats = new ScriptStats(5, 6, 7, new TimeSeries(10, 20, 30, 40), new TimeSeries(100, 200, 300, 400)); + final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + + builder.startObject(); + for (var it = stats.toXContentChunked(ToXContent.EMPTY_PARAMS); it.hasNext();) { + it.next().toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endObject(); + String expected = """ + { + "script" : { + "compilations" : 5, + "cache_evictions" : 6, + "compilation_limit_triggered" : 7, + "compilations_history" : { + "5m" : 10, + "15m" : 20, + "24h" : 30 + }, + "cache_evictions_history" : { + "5m" : 100, + "15m" : 200, + "24h" : 300 + }, + "contexts" : [ ] + } + }"""; + assertThat(Strings.toString(builder), equalTo(expected)); + } + public void testSerializeEmptyTimeSeries() throws IOException { ScriptContextStats stats = new ScriptContextStats("c", 3333, new TimeSeries(1111), new TimeSeries(2222)); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java index ef19de6a134b0..faf16443cab7c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceSingleNodeTests.java @@ -180,6 +180,8 @@ public class SearchServiceSingleNodeTests extends ESSingleNodeTestCase { + private static final int SEARCH_POOL_SIZE = 10; + @Override protected boolean resetNodeAfterTest() { return true; @@ -265,7 +267,10 @@ public void onQueryPhase(SearchContext context, long tookInNanos) { @Override protected Settings nodeSettings() { - return Settings.builder().put("search.default_search_timeout", "5s").build(); + return Settings.builder() + .put("search.default_search_timeout", "5s") + .put("thread_pool.search.size", SEARCH_POOL_SIZE) // customized search pool size, reconfiguring at runtime is unsupported + .build(); } public void testClearOnClose() { @@ -2148,6 +2153,7 @@ public void onFailure(Exception exc) { CountDownLatch latch = new CountDownLatch(1); shardRequest.source().query(new MatchNoneQueryBuilder()); service.executeQueryPhase(shardRequest, task, new ActionListener<>() { + @Override public void onResponse(SearchPhaseResult result) { try { @@ -2748,8 +2754,11 @@ public void testEnableSearchWorkerThreads() throws IOException { public void testSlicingBehaviourForParallelCollection() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY); ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH); - final int configuredMaxPoolSize = 10; - executor.setMaximumPoolSize(configuredMaxPoolSize); // We set this explicitly to be independent of CPU cores. + + // We configure the executor pool size explicitly in nodeSettings to be independent of CPU cores + assert String.valueOf(SEARCH_POOL_SIZE).equals(node().settings().get("thread_pool.search.size")) + : "Unexpected thread_pool.search.size"; + int numDocs = randomIntBetween(50, 100); for (int i = 0; i < numDocs; i++) { prepareIndex("index").setId(String.valueOf(i)).setSource("field", "value").get(); @@ -2782,7 +2791,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { final int maxPoolSize = executor.getMaximumPoolSize(); assertEquals( "Sanity check to ensure this isn't the default of 1 when pool size is unset", - configuredMaxPoolSize, + SEARCH_POOL_SIZE, maxPoolSize ); @@ -2812,7 +2821,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { final int maxPoolSize = executor.getMaximumPoolSize(); assertEquals( "Sanity check to ensure this isn't the default of 1 when pool size is unset", - configuredMaxPoolSize, + SEARCH_POOL_SIZE, maxPoolSize ); @@ -2903,7 +2912,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { final int maxPoolSize = executor.getMaximumPoolSize(); assertEquals( "Sanity check to ensure this isn't the default of 1 when pool size is unset", - configuredMaxPoolSize, + SEARCH_POOL_SIZE, maxPoolSize ); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index d041121b8a96b..7c89c106a7c69 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -9,6 +9,8 @@ package org.elasticsearch.search; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; @@ -51,6 +53,7 @@ import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.MinAndMax; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.MockLog; import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.IOException; @@ -59,6 +62,7 @@ import java.util.function.BiFunction; import java.util.function.Predicate; +import static org.elasticsearch.common.Strings.format; import static org.elasticsearch.search.SearchService.maybeWrapListenerForStackTrace; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.not; @@ -125,6 +129,7 @@ public Type getType() { } public void testMaybeWrapListenerForStackTrace() { + ShardId shardId = new ShardId("index", "index", 0); // Tests that the same listener has stack trace if is not wrapped or does not have stack trace if it is wrapped. AtomicBoolean isWrapped = new AtomicBoolean(false); ActionListener listener = new ActionListener<>() { @@ -146,11 +151,86 @@ public void onFailure(Exception e) { e.fillInStackTrace(); assertThat(e.getStackTrace().length, is(not(0))); listener.onFailure(e); - listener = maybeWrapListenerForStackTrace(listener, TransportVersion.current(), threadPool); + listener = maybeWrapListenerForStackTrace(listener, TransportVersion.current(), "node", shardId, 123L, threadPool); isWrapped.set(true); listener.onFailure(e); } + public void testMaybeWrapListenerForStackTraceDebugLog() { + final String nodeId = "node"; + final String index = "index"; + ShardId shardId = new ShardId(index, index, 0); + final long taskId = 123L; + + try (var mockLog = MockLog.capture(SearchService.class)) { + Configurator.setLevel(SearchService.class, Level.DEBUG); + final String exceptionMessage = "test exception message"; + mockLog.addExpectation( + new MockLog.ExceptionSeenEventExpectation( + format("\"[%s]%s: failed to execute search request for task [%d]\" and an exception logged", nodeId, shardId, taskId), + SearchService.class.getCanonicalName(), + Level.DEBUG, // We will throw a 400-level exception, so it should only be logged at the debug level + format("[%s]%s: failed to execute search request for task [%d]", nodeId, shardId, taskId), + IllegalArgumentException.class, + exceptionMessage + ) + ); + + // Tests the listener has logged if it is wrapped + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(SearchPhaseResult searchPhaseResult) { + // noop - we only care about failure scenarios + } + + @Override + public void onFailure(Exception e) { + mockLog.assertAllExpectationsMatched(); + } + }; + IllegalArgumentException e = new IllegalArgumentException(exceptionMessage); // 400-level exception + listener = maybeWrapListenerForStackTrace(listener, TransportVersion.current(), nodeId, shardId, taskId, threadPool); + listener.onFailure(e); + } + } + + public void testMaybeWrapListenerForStackTraceWarnLog() { + final String nodeId = "node"; + final String index = "index"; + ShardId shardId = new ShardId(index, index, 0); + final long taskId = 123L; + + try (var mockLog = MockLog.capture(SearchService.class)) { + final String exceptionMessage = "test exception message"; + mockLog.addExpectation( + new MockLog.ExceptionSeenEventExpectation( + format("\"[%s]%s: failed to execute search request for task [%d]\" and an exception logged", nodeId, shardId, taskId), + SearchService.class.getCanonicalName(), + Level.WARN, // We will throw a 500-level exception, so it should be logged at the warn level + format("[%s]%s: failed to execute search request for task [%d]", nodeId, shardId, taskId), + IllegalStateException.class, + exceptionMessage + ) + ); + + // Tests the listener has logged if it is wrapped + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(SearchPhaseResult searchPhaseResult) { + // noop - we only care about failure scenarios + } + + @Override + public void onFailure(Exception e) { + mockLog.assertAllExpectationsMatched(); + } + }; + IllegalStateException e = new IllegalStateException(exceptionMessage); // 500-level exception + listener = maybeWrapListenerForStackTrace(listener, TransportVersion.current(), nodeId, shardId, taskId, threadPool); + listener.onFailure(e); + } + } + private void doTestCanMatch( SearchRequest searchRequest, SortField sortField, diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java index 71b93888ba243..21fe71d6f6421 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java @@ -333,10 +333,7 @@ private void testRandomCase(boolean forceMerge, boolean missingBucket, int index final SortedDocsProducer docsProducer = sources[0].createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()); for (LeafReaderContext leafReaderContext : reader.leaves()) { if (docsProducer != null && withProducer) { - assertEquals( - DocIdSet.EMPTY, - docsProducer.processLeaf(new MatchAllDocsQuery(), queue, leafReaderContext, false) - ); + assertEquals(DocIdSet.EMPTY, docsProducer.processLeaf(queue, leafReaderContext, false)); } else { final LeafBucketCollector leafCollector = new LeafBucketCollector() { @Override diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index cdbf4cdff15a7..9fef56ba16396 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.collapse.CollapseBuilderTests; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; @@ -1016,6 +1017,40 @@ public void testSupportsParallelCollection() { searchSourceBuilder.aggregation(new TermsAggregationBuilder("terms")); assertFalse(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); } + { + SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); + searchSourceBuilder.aggregation(new TopHitsAggregationBuilder("tophits")); + assertTrue(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + } + { + SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); + searchSourceBuilder.aggregation(new TopHitsAggregationBuilder("tophits").sort("_score")); + assertTrue(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + } + { + SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); + searchSourceBuilder.aggregation(new TopHitsAggregationBuilder("tophits").sort(SortBuilders.fieldSort("field"))); + assertTrue(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + } + { + SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("terms").subAggregation(new TopHitsAggregationBuilder("tophits"))); + assertFalse(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + } + { + SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); + searchSourceBuilder.aggregation(new TopHitsAggregationBuilder("tophits").subAggregation(new TermsAggregationBuilder("terms"))); + assertFalse(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + } + { + SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); + searchSourceBuilder.aggregation( + new TopHitsAggregationBuilder("terms").sort( + SortBuilders.scriptSort(new Script("id"), randomFrom(ScriptSortBuilder.ScriptSortType.values())) + ) + ); + assertTrue(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + } { SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); searchSourceBuilder.collapse(CollapseBuilderTests.randomCollapseBuilder()); @@ -1035,7 +1070,7 @@ public void testSupportsParallelCollection() { ScriptSortBuilder.ScriptSortType.NUMBER ).order(randomFrom(SortOrder.values())) ); - assertFalse(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); + assertTrue(searchSourceBuilder.supportsParallelCollection(fieldCardinality)); } { SearchSourceBuilder searchSourceBuilder = newSearchSourceBuilder.get(); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 3699cdee3912b..8e625231dbea9 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -577,10 +577,10 @@ public void testPreTagsWithoutPostTags() throws IOException { public void testInvalidMaxAnalyzedOffset() throws IOException { XContentParseException e = expectParseThrows( XContentParseException.class, - "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, 0) + "}" + "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, -2) + "}" ); assertThat(e.getMessage(), containsString("[highlight] failed to parse field [" + MAX_ANALYZED_OFFSET_FIELD.toString() + "]")); - assertThat(e.getCause().getMessage(), containsString("[max_analyzed_offset] must be a positive integer")); + assertThat(e.getCause().getMessage(), containsString("[max_analyzed_offset] must be a positive integer, or -1")); } /** diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java index 3bf9514cad547..2b250ce772368 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.Scorable; @@ -37,14 +38,29 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.CharsRefBuilder; +import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchShardTask; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.text.Text; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.test.TestSearchContext; +import org.hamcrest.Matchers; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -258,6 +274,119 @@ private TestSearchContext createSearchContext(Query query, int size) throws IOEx return context; } + public void testSuggestOnlyWithTimeout() throws Exception { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().suggest(new SuggestBuilder()); + try (SearchContext context = createSearchContextWithSuggestTimeout(searchSourceBuilder)) { + assertTrue(context.hasOnlySuggest()); + QueryPhase.execute(context); + assertTrue(context.queryResult().searchTimedOut()); + assertEquals(1, context.queryResult().suggest().size()); + assertEquals(0, context.queryResult().suggest().getSuggestion("suggestion").getEntries().size()); + assertNotNull(context.queryResult().topDocs()); + assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value); + } + } + + public void testSuggestAndQueryWithSuggestTimeout() throws Exception { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().suggest(new SuggestBuilder()).query(new MatchAllQueryBuilder()); + try (SearchContext context = createSearchContextWithSuggestTimeout(searchSourceBuilder)) { + context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); + assertFalse(context.hasOnlySuggest()); + QueryPhase.execute(context); + assertThat(context.queryResult().topDocs().topDocs.totalHits.value, Matchers.greaterThan(0L)); + assertTrue(context.queryResult().searchTimedOut()); + assertEquals(1, context.queryResult().suggest().size()); + assertEquals(0, context.queryResult().suggest().getSuggestion("suggestion").getEntries().size()); + } + } + + private TestSearchContext createSearchContextWithSuggestTimeout(SearchSourceBuilder searchSourceBuilder) throws IOException { + ContextIndexSearcher contextIndexSearcher = newContextSearcher(reader); + SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); + suggestionSearchContext.addSuggestion("suggestion", new TestSuggestionContext(new TestSuggester(contextIndexSearcher), null)); + TestSearchContext context = new TestSearchContext(null, indexShard, contextIndexSearcher) { + @Override + public SuggestionSearchContext suggest() { + return suggestionSearchContext; + } + + @Override + public ShardSearchRequest request() { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.allowPartialSearchResults(true); + searchRequest.source(searchSourceBuilder); + return new ShardSearchRequest( + OriginalIndices.NONE, + searchRequest, + indexShard.shardId(), + 0, + 1, + AliasFilter.EMPTY, + 1F, + 0, + null + ); + } + }; + context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); + return context; + } + + private static final class TestSuggester extends Suggester { + private final ContextIndexSearcher contextIndexSearcher; + + TestSuggester(ContextIndexSearcher contextIndexSearcher) { + this.contextIndexSearcher = contextIndexSearcher; + } + + @Override + protected TestSuggestion innerExecute( + String name, + TestSuggestionContext suggestion, + IndexSearcher searcher, + CharsRefBuilder spare + ) { + contextIndexSearcher.throwTimeExceededException(); + throw new AssertionError("should have thrown TimeExceededException"); + } + + @Override + protected TestSuggestion emptySuggestion(String name, TestSuggestionContext suggestion, CharsRefBuilder spare) { + return new TestSuggestion(); + } + } + + private static final class TestSuggestionContext extends SuggestionSearchContext.SuggestionContext { + TestSuggestionContext(Suggester suggester, SearchExecutionContext searchExecutionContext) { + super(suggester, searchExecutionContext); + } + } + + private static final class TestSuggestion extends Suggest.Suggestion< + Suggest.Suggestion.Entry> { + TestSuggestion() { + super("suggestion", 10); + } + + @Override + protected Entry newEntry(StreamInput in) { + return new TestSuggestionEntry(); + } + + @Override + public String getWriteableName() { + return "suggestion"; + } + } + + private static final class TestSuggestionEntry extends Suggest.Suggestion.Entry { + @Override + protected Option newOption(StreamInput in) { + return new Option(new Text("text"), 1f) { + }; + } + } + private static class ScoreAndDoc extends Scorable { float score; int doc = -1; diff --git a/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java b/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java index aa719ff1d6ae8..6d23622f86245 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java @@ -9,35 +9,39 @@ package org.elasticsearch.search.vectors; +import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.queries.function.FunctionScoreQuery; +import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; +import org.elasticsearch.index.codec.Elasticsearch816Codec; +import org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat; +import org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat; +import org.elasticsearch.index.mapper.vectors.VectorSimilarityFloatValueSource; import org.elasticsearch.search.profile.query.QueryProfiler; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.PriorityQueue; -import java.util.stream.Collectors; - -import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -58,50 +62,45 @@ public void testRescoreDocs() throws Exception { // Use a RescoreKnnVectorQuery with a match all query, to ensure we get scoring of 1 from the inner query // and thus we're rescoring the top k docs. float[] queryVector = randomVector(numDims); + Query innerQuery; + if (randomBoolean()) { + innerQuery = new KnnFloatVectorQuery(FIELD_NAME, queryVector, (int) (k * randomFloatBetween(1.0f, 10.0f, true))); + } else { + innerQuery = new MatchAllDocsQuery(); + } RescoreKnnVectorQuery rescoreKnnVectorQuery = new RescoreKnnVectorQuery( FIELD_NAME, queryVector, VectorSimilarityFunction.COSINE, k, - new MatchAllDocsQuery() + innerQuery ); IndexSearcher searcher = newSearcher(reader, true, false); - TopDocs docs = searcher.search(rescoreKnnVectorQuery, numDocs); - Map rescoredDocs = Arrays.stream(docs.scoreDocs) - .collect(Collectors.toMap(scoreDoc -> scoreDoc.doc, scoreDoc -> scoreDoc.score)); - - assertThat(rescoredDocs.size(), equalTo(k)); - - Collection rescoredScores = new HashSet<>(rescoredDocs.values()); - - // Collect all docs sequentially, and score them using the similarity function to get the top K scores - PriorityQueue topK = new PriorityQueue<>((o1, o2) -> Float.compare(o2, o1)); - - for (LeafReaderContext leafReaderContext : reader.leaves()) { - FloatVectorValues vectorValues = leafReaderContext.reader().getFloatVectorValues(FIELD_NAME); - while (vectorValues.nextDoc() != NO_MORE_DOCS) { - float[] vectorData = vectorValues.vectorValue(); - float score = VectorSimilarityFunction.COSINE.compare(queryVector, vectorData); - topK.add(score); - int docId = vectorValues.docID(); - // If the doc has been retrieved from the RescoreKnnVectorQuery, check the score is the same and remove it - // to ensure we found them all - if (rescoredDocs.containsKey(docId)) { - assertThat(rescoredDocs.get(docId), equalTo(score)); - rescoredDocs.remove(docId); - } - } - } - - assertThat(rescoredDocs.size(), equalTo(0)); + TopDocs rescoredDocs = searcher.search(rescoreKnnVectorQuery, numDocs); + assertThat(rescoredDocs.scoreDocs.length, equalTo(k)); - // Check top scoring docs are contained in rescored docs - for (int i = 0; i < k; i++) { - Float topScore = topK.poll(); - if (rescoredScores.contains(topScore) == false) { - fail("Top score " + topScore + " not contained in rescored doc scores " + rescoredScores); + // Get real scores + DoubleValuesSource valueSource = new VectorSimilarityFloatValueSource( + FIELD_NAME, + queryVector, + VectorSimilarityFunction.COSINE + ); + FunctionScoreQuery functionScoreQuery = new FunctionScoreQuery(new MatchAllDocsQuery(), valueSource); + TopDocs realScoreTopDocs = searcher.search(functionScoreQuery, numDocs); + + int i = 0; + ScoreDoc[] realScoreDocs = realScoreTopDocs.scoreDocs; + for (ScoreDoc rescoreDoc : rescoredDocs.scoreDocs) { + // There are docs that won't be found in the rescored search, but every doc found must be in the same order + // and have the same score + while (i < realScoreDocs.length && realScoreDocs[i].doc != rescoreDoc.doc) { + i++; + } + if (i >= realScoreDocs.length) { + fail("Rescored doc not found in real score docs"); } + assertThat("Real score is not the same as rescored score", rescoreDoc.score, equalTo(realScoreDocs[i].score)); } } } @@ -203,16 +202,33 @@ public void profile(QueryProfiler queryProfiler) { } private static void addRandomDocuments(int numDocs, Directory d, int numDims) throws IOException { + IndexWriterConfig iwc = new IndexWriterConfig(); + // Pick codec from quantized vector formats to ensure scores use real scores when using knn rescore + KnnVectorsFormat format = randomFrom( + new ES818BinaryQuantizedVectorsFormat(), + new ES818HnswBinaryQuantizedVectorsFormat(), + new ES813Int8FlatVectorFormat(), + new ES813Int8FlatVectorFormat(), + new ES814HnswScalarQuantizedVectorsFormat() + ); + iwc.setCodec(new Elasticsearch816Codec(randomFrom(Zstd814StoredFieldsFormat.Mode.values())) { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return format; + } + }); try (IndexWriter w = new IndexWriter(d, newIndexWriterConfig())) { for (int i = 0; i < numDocs; i++) { Document document = new Document(); float[] vector = randomVector(numDims); - KnnFloatVectorField vectorField = new KnnFloatVectorField(FIELD_NAME, vector); + KnnFloatVectorField vectorField = new KnnFloatVectorField(FIELD_NAME, vector, VectorSimilarityFunction.COSINE); document.add(vectorField); w.addDocument(document); + if (randomBoolean() && (i % 10 == 0)) { + w.commit(); + } } w.commit(); - w.forceMerge(1); } } } diff --git a/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java b/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java index cb266c58d70d5..3ec248e0d8d9a 100644 --- a/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java @@ -290,7 +290,12 @@ public void testLogsSlowInboundProcessing() throws Exception { ); BytesStreamOutput byteData = new BytesStreamOutput(); TaskId.EMPTY_TASK_ID.writeTo(byteData); - TransportVersion.writeVersion(remoteVersion, byteData); + // simulate bytes of a transport handshake: vInt transport version then release version string + try (var payloadByteData = new BytesStreamOutput()) { + TransportVersion.writeVersion(remoteVersion, payloadByteData); + payloadByteData.writeString(randomIdentifier()); + byteData.writeBytesReference(payloadByteData.bytes()); + } final InboundMessage requestMessage = new InboundMessage( requestHeader, ReleasableBytesReference.wrap(byteData.bytes()), diff --git a/server/src/test/java/org/elasticsearch/transport/TransportHandshakerRawMessageTests.java b/server/src/test/java/org/elasticsearch/transport/TransportHandshakerRawMessageTests.java index b4c791c7a04b3..6662aadd51fbb 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportHandshakerRawMessageTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportHandshakerRawMessageTests.java @@ -39,6 +39,7 @@ public class TransportHandshakerRawMessageTests extends ESSingleNodeTestCase { public void testV7Handshake() throws Exception { final BytesRef handshakeRequestBytes; final var requestId = randomNonNegativeLong(); + final var requestNodeTransportVersionId = TransportVersionUtils.randomCompatibleVersion(random()).id(); try (var outputStream = new BytesStreamOutput()) { outputStream.setTransportVersion(TransportHandshaker.V7_HANDSHAKE_VERSION); outputStream.writeLong(requestId); @@ -50,7 +51,6 @@ public void testV7Handshake() throws Exception { outputStream.writeString("internal:tcp/handshake"); outputStream.writeByte((byte) 0); // no parent task ID; - final var requestNodeTransportVersionId = TransportVersionUtils.randomCompatibleVersion(random()).id(); assertThat(requestNodeTransportVersionId, allOf(greaterThanOrEqualTo(1 << 22), lessThan(1 << 28))); // 4-byte vInt outputStream.writeByte((byte) 4); // payload length outputStream.writeVInt(requestNodeTransportVersionId); @@ -80,7 +80,7 @@ public void testV7Handshake() throws Exception { assertEquals((byte) 0, inputStream.readByte()); // no request headers assertEquals((byte) 0, inputStream.readByte()); // no response headers inputStream.setTransportVersion(TransportHandshaker.V7_HANDSHAKE_VERSION); - assertEquals(TransportVersion.current().id(), inputStream.readVInt()); + assertEquals(requestNodeTransportVersionId, inputStream.readVInt()); assertEquals(-1, inputStream.read()); } } @@ -88,6 +88,7 @@ public void testV7Handshake() throws Exception { public void testV8Handshake() throws Exception { final BytesRef handshakeRequestBytes; final var requestId = randomNonNegativeLong(); + final var requestNodeTransportVersionId = TransportVersionUtils.randomCompatibleVersion(random()).id(); try (var outputStream = new BytesStreamOutput()) { outputStream.setTransportVersion(TransportHandshaker.V8_HANDSHAKE_VERSION); outputStream.writeLong(requestId); @@ -100,7 +101,6 @@ public void testV8Handshake() throws Exception { outputStream.writeString("internal:tcp/handshake"); outputStream.writeByte((byte) 0); // no parent task ID; - final var requestNodeTransportVersionId = TransportVersionUtils.randomCompatibleVersion(random()).id(); assertThat(requestNodeTransportVersionId, allOf(greaterThanOrEqualTo(1 << 22), lessThan(1 << 28))); // 4-byte vInt outputStream.writeByte((byte) 4); // payload length outputStream.writeVInt(requestNodeTransportVersionId); @@ -131,7 +131,7 @@ public void testV8Handshake() throws Exception { assertEquals((byte) 0, inputStream.readByte()); // no request headers assertEquals((byte) 0, inputStream.readByte()); // no response headers inputStream.setTransportVersion(TransportHandshaker.V8_HANDSHAKE_VERSION); - assertEquals(TransportVersion.current().id(), inputStream.readVInt()); + assertEquals(requestNodeTransportVersionId, inputStream.readVInt()); assertEquals(-1, inputStream.read()); } } @@ -139,6 +139,7 @@ public void testV8Handshake() throws Exception { public void testV9Handshake() throws Exception { final BytesRef handshakeRequestBytes; final var requestId = randomNonNegativeLong(); + final var requestNodeTransportVersionId = TransportVersionUtils.randomCompatibleVersion(random()).id(); try (var outputStream = new BytesStreamOutput()) { outputStream.setTransportVersion(TransportHandshaker.V9_HANDSHAKE_VERSION); outputStream.writeLong(requestId); @@ -150,7 +151,6 @@ public void testV9Handshake() throws Exception { outputStream.writeString("internal:tcp/handshake"); outputStream.writeByte((byte) 0); // no parent task ID; - final var requestNodeTransportVersionId = TransportVersionUtils.randomCompatibleVersion(random()).id(); assertThat(requestNodeTransportVersionId, allOf(greaterThanOrEqualTo(1 << 22), lessThan(1 << 28))); // 4-byte vInt final var releaseVersionLength = between(0, 127 - 5); // so that its length, and the length of the payload, is a one-byte vInt final var requestNodeReleaseVersion = randomAlphaOfLength(releaseVersionLength); @@ -184,7 +184,7 @@ public void testV9Handshake() throws Exception { assertEquals((byte) 0, inputStream.readByte()); // no request headers assertEquals((byte) 0, inputStream.readByte()); // no response headers inputStream.setTransportVersion(TransportHandshaker.V9_HANDSHAKE_VERSION); - assertEquals(TransportVersion.current().id(), inputStream.readVInt()); + assertEquals(requestNodeTransportVersionId, inputStream.readVInt()); assertEquals(Build.current().version(), inputStream.readString()); assertEquals(-1, inputStream.read()); } diff --git a/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java b/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java index 018c0f2195e8c..2516d4ecfd5e5 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java @@ -8,23 +8,33 @@ */ package org.elasticsearch.transport; +import org.apache.logging.log4j.Level; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.test.TransportVersionUtils; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.TestThreadPool; import java.io.IOException; import java.util.Collections; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.transport.TransportHandshaker.V8_18_FIRST_VERSION; +import static org.elasticsearch.transport.TransportHandshaker.getDeprecationMessage; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -91,6 +101,84 @@ public void testHandshakeRequestAndResponse() throws IOException { assertEquals(TransportVersion.current(), versionFuture.actionGet()); } + @TestLogging(reason = "testing WARN logging", value = "org.elasticsearch.transport.TransportHandshaker:WARN") + public void testIncompatibleHandshakeRequest() throws Exception { + var remoteVersion = getRandomIncompatibleTransportVersion(); + TransportHandshaker.HandshakeRequest handshakeRequest = new TransportHandshaker.HandshakeRequest(remoteVersion, randomIdentifier()); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + bytesStreamOutput.setTransportVersion(HANDSHAKE_REQUEST_VERSION); + handshakeRequest.writeTo(bytesStreamOutput); + StreamInput input = bytesStreamOutput.bytes().streamInput(); + input.setTransportVersion(HANDSHAKE_REQUEST_VERSION); + + final TestTransportChannel channel; + if (handshakeRequest.transportVersion.onOrAfter(TransportVersions.MINIMUM_COMPATIBLE)) { + + final PlainActionFuture responseFuture = new PlainActionFuture<>(); + channel = new TestTransportChannel(responseFuture); + + // we fall back to the best known version + MockLog.assertThatLogger(() -> { + try { + handshaker.handleHandshake(channel, randomNonNegativeLong(), input); + } catch (IOException e) { + throw new AssertionError(e); + } + }, + TransportHandshaker.class, + new MockLog.SeenEventExpectation( + "warning", + TransportHandshaker.class.getCanonicalName(), + Level.WARN, + Strings.format( + """ + Negotiating transport handshake with remote node with version [%s/%s] received on [*] which appears to be from \ + a chronologically-older release with a numerically-newer version compared to this node's version [%s/%s]. \ + Upgrading to a chronologically-older release may not work reliably and is not recommended. Falling back to \ + transport protocol version [%s].""", + handshakeRequest.transportVersion.toReleaseVersion(), + handshakeRequest.transportVersion, + Build.current().version(), + TransportVersion.current(), + handshakeRequest.transportVersion.bestKnownVersion() + ) + ) + ); + + assertTrue(responseFuture.isDone()); + assertEquals( + handshakeRequest.transportVersion.bestKnownVersion(), + asInstanceOf(TransportHandshaker.HandshakeResponse.class, responseFuture.result()).getTransportVersion() + ); + + } else { + channel = new TestTransportChannel(ActionListener.running(() -> fail("should not complete"))); + + MockLog.assertThatLogger( + () -> assertThat( + expectThrows(IllegalStateException.class, () -> handshaker.handleHandshake(channel, randomNonNegativeLong(), input)) + .getMessage(), + allOf( + containsString("Rejecting unreadable transport handshake"), + containsString( + "[" + handshakeRequest.transportVersion.toReleaseVersion() + "/" + handshakeRequest.transportVersion + "]" + ), + containsString("[" + Build.current().version() + "/" + TransportVersion.current() + "]"), + containsString("which has an incompatible wire format") + ) + ), + TransportHandshaker.class, + new MockLog.SeenEventExpectation( + "warning", + TransportHandshaker.class.getCanonicalName(), + Level.WARN, + "Rejecting unreadable transport handshake * incompatible wire format." + ) + ); + } + assertDeprecationMessageIsLogged(remoteVersion, remoteVersion.toReleaseVersion(), channel); + } + public void testHandshakeResponseFromOlderNode() throws Exception { final PlainActionFuture versionFuture = new PlainActionFuture<>(); final long reqId = randomNonNegativeLong(); @@ -100,10 +188,95 @@ public void testHandshakeResponseFromOlderNode() throws Exception { assertFalse(versionFuture.isDone()); final var remoteVersion = TransportVersionUtils.randomCompatibleVersion(random()); - handler.handleResponse(new TransportHandshaker.HandshakeResponse(remoteVersion, randomIdentifier())); + var releaseVersion = randomIdentifier(); + handler.handleResponse(new TransportHandshaker.HandshakeResponse(remoteVersion, releaseVersion)); assertTrue(versionFuture.isDone()); assertEquals(remoteVersion, versionFuture.result()); + + assertDeprecationMessageIsLogged(remoteVersion, releaseVersion, channel); + } + + @TestLogging(reason = "testing WARN logging", value = "org.elasticsearch.transport.TransportHandshaker:WARN") + public void testHandshakeResponseFromOlderNodeWithPatchedProtocol() throws Exception { + final PlainActionFuture versionFuture = new PlainActionFuture<>(); + final long reqId = randomNonNegativeLong(); + handshaker.sendHandshake(reqId, node, channel, SAFE_AWAIT_TIMEOUT, versionFuture); + TransportResponseHandler handler = handshaker.removeHandlerForHandshake(reqId); + + assertFalse(versionFuture.isDone()); + + var remoteVersion = getRandomIncompatibleTransportVersion(); + var releaseVersion = randomIdentifier(); + final var handshakeResponse = new TransportHandshaker.HandshakeResponse(remoteVersion, releaseVersion); + + if (remoteVersion.onOrAfter(TransportVersions.MINIMUM_COMPATIBLE)) { + // we fall back to the best known version + MockLog.assertThatLogger( + () -> handler.handleResponse(handshakeResponse), + TransportHandshaker.class, + new MockLog.SeenEventExpectation( + "warning", + TransportHandshaker.class.getCanonicalName(), + Level.WARN, + Strings.format( + """ + Negotiating transport handshake with remote node with version [%s/%s] received on [*] which appears to be from \ + a chronologically-older release with a numerically-newer version compared to this node's version [%s/%s]. \ + Upgrading to a chronologically-older release may not work reliably and is not recommended. Falling back to \ + transport protocol version [%s].""", + handshakeResponse.getReleaseVersion(), + handshakeResponse.getTransportVersion(), + Build.current().version(), + TransportVersion.current(), + remoteVersion.bestKnownVersion() + ) + ) + ); + + assertTrue(versionFuture.isDone()); + assertEquals(remoteVersion.bestKnownVersion(), versionFuture.result()); + } else { + MockLog.assertThatLogger( + () -> handler.handleResponse(handshakeResponse), + TransportHandshaker.class, + new MockLog.SeenEventExpectation( + "warning", + TransportHandshaker.class.getCanonicalName(), + Level.WARN, + "Rejecting unreadable transport handshake * incompatible wire format." + ) + ); + + assertTrue(versionFuture.isDone()); + assertThat( + expectThrows(ExecutionException.class, IllegalStateException.class, versionFuture::result).getMessage(), + allOf( + containsString("Rejecting unreadable transport handshake"), + containsString("[" + handshakeResponse.getReleaseVersion() + "/" + handshakeResponse.getTransportVersion() + "]"), + containsString("[" + Build.current().version() + "/" + TransportVersion.current() + "]"), + containsString("which has an incompatible wire format") + ) + ); + } + assertDeprecationMessageIsLogged(remoteVersion, releaseVersion, channel); + } + + private void assertDeprecationMessageIsLogged(TransportVersion remoteVersion, String remoteReleaseVersion, Object channel) { + if (remoteVersion.onOrAfter(TransportVersions.MINIMUM_COMPATIBLE) && remoteVersion.before(V8_18_FIRST_VERSION)) { + assertCriticalWarnings(getDeprecationMessage(TransportVersion.current(), remoteVersion, remoteReleaseVersion, channel)); + } + } + + private static TransportVersion getRandomIncompatibleTransportVersion() { + return randomBoolean() + // either older than MINIMUM_COMPATIBLE + ? new TransportVersion(between(1, TransportVersions.MINIMUM_COMPATIBLE.id() - 1)) + // or between MINIMUM_COMPATIBLE and current but not known + : randomValueOtherThanMany( + TransportVersion::isKnown, + () -> new TransportVersion(between(TransportVersions.MINIMUM_COMPATIBLE.id(), TransportVersion.current().id())) + ); } public void testHandshakeResponseFromNewerNode() throws Exception { @@ -144,7 +317,7 @@ public void testHandshakeRequestFutureVersionsCompatibility() throws IOException TaskId.EMPTY_TASK_ID.writeTo(lengthCheckingHandshake); TaskId.EMPTY_TASK_ID.writeTo(futureHandshake); try (BytesStreamOutput internalMessage = new BytesStreamOutput()) { - Version.writeVersion(Version.CURRENT, internalMessage); + internalMessage.writeVInt(TransportVersion.current().id() + between(0, 100)); lengthCheckingHandshake.writeBytesReference(internalMessage.bytes()); internalMessage.write(new byte[1024]); futureHandshake.writeBytesReference(internalMessage.bytes()); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java index c686329c4154c..a0d7de67c5092 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.core.Releasable; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -373,39 +372,32 @@ public void testRejectsMismatchedBuildHash() { assertFalse(transportServiceA.nodeConnected(discoveryNode)); } - @SuppressForbidden(reason = "Sets property for testing") public void testAcceptsMismatchedServerlessBuildHash() { assumeTrue("Current build needs to be a snapshot", Build.current().isSnapshot()); - assumeTrue("Security manager needs to be disabled", System.getSecurityManager() == null); - System.setProperty("es.serverless", Boolean.TRUE.toString()); // security manager blocks this - try { - final DisruptingTransportInterceptor transportInterceptorA = new DisruptingTransportInterceptor(); - final DisruptingTransportInterceptor transportInterceptorB = new DisruptingTransportInterceptor(); - transportInterceptorA.setModifyBuildHash(true); - transportInterceptorB.setModifyBuildHash(true); - final Settings settings = Settings.builder() - .put("cluster.name", "a") - .put(IGNORE_DESERIALIZATION_ERRORS_SETTING.getKey(), true) // suppress assertions to test production error-handling - .build(); - final TransportService transportServiceA = startServices( - "TS_A", - settings, - TransportVersion.current(), - VersionInformation.CURRENT, - transportInterceptorA - ); - final TransportService transportServiceB = startServices( - "TS_B", - settings, - TransportVersion.current(), - VersionInformation.CURRENT, - transportInterceptorB - ); - AbstractSimpleTransportTestCase.connectToNode(transportServiceA, transportServiceB.getLocalNode(), TestProfiles.LIGHT_PROFILE); - assertTrue(transportServiceA.nodeConnected(transportServiceB.getLocalNode())); - } finally { - System.clearProperty("es.serverless"); - } + final DisruptingTransportInterceptor transportInterceptorA = new DisruptingTransportInterceptor(); + final DisruptingTransportInterceptor transportInterceptorB = new DisruptingTransportInterceptor(); + transportInterceptorA.setModifyBuildHash(true); + transportInterceptorB.setModifyBuildHash(true); + final Settings settings = Settings.builder() + .put("cluster.name", "a") + .put(IGNORE_DESERIALIZATION_ERRORS_SETTING.getKey(), true) // suppress assertions to test production error-handling + .build(); + final TransportService transportServiceA = startServices( + "TS_A", + settings, + TransportVersion.current(), + VersionInformation.CURRENT, + transportInterceptorA + ); + final TransportService transportServiceB = startServices( + "TS_B", + settings, + TransportVersion.current(), + VersionInformation.CURRENT, + transportInterceptorB + ); + AbstractSimpleTransportTestCase.connectToNode(transportServiceA, transportServiceB.getLocalNode(), TestProfiles.LIGHT_PROFILE); + assertTrue(transportServiceA.nodeConnected(transportServiceB.getLocalNode())); } public void testAcceptsMismatchedBuildHashFromDifferentVersion() { diff --git a/settings.gradle b/settings.gradle index 8ee39ace46fdd..27037b8934887 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,7 +14,7 @@ pluginManagement { } plugins { - id "com.gradle.develocity" version "3.18.1" + id "com.gradle.develocity" version "3.19.2" id 'elasticsearch.java-toolchain' } diff --git a/test/external-modules/apm-integration/build.gradle b/test/external-modules/apm-integration/build.gradle index 0be702efd736b..03b90cb0d8ec3 100644 --- a/test/external-modules/apm-integration/build.gradle +++ b/test/external-modules/apm-integration/build.gradle @@ -17,7 +17,7 @@ tasks.named("test").configure { } tasks.named('javaRestTest').configure { - it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.snapshotBuild } } dependencies { diff --git a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java index e974c31bf5c08..07141846dde4d 100644 --- a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java +++ b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java @@ -127,7 +127,7 @@ public void testApmIntegration() throws Exception { var completed = finished.await(30, TimeUnit.SECONDS); var remainingAssertions = Stream.concat(valueAssertions.keySet().stream(), histogramAssertions.keySet().stream()) - .collect(Collectors.joining()); + .collect(Collectors.joining(",")); assertTrue("Timeout when waiting for assertions to complete. Remaining assertions to match: " + remainingAssertions, completed); } diff --git a/test/external-modules/build.gradle b/test/external-modules/build.gradle index 61acfa917bc7f..5dcf92738d2cc 100644 --- a/test/external-modules/build.gradle +++ b/test/external-modules/build.gradle @@ -6,7 +6,7 @@ subprojects { esplugin { name = it.name - licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile = rootProject.file('NOTICE.txt') + licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile + noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } } diff --git a/test/external-modules/delayed-aggs/build.gradle b/test/external-modules/delayed-aggs/build.gradle index 0c86e233930a7..92ee2dddd89c4 100644 --- a/test/external-modules/delayed-aggs/build.gradle +++ b/test/external-modules/delayed-aggs/build.gradle @@ -8,11 +8,7 @@ */ import org.elasticsearch.gradle.internal.info.BuildParams -apply plugin: 'elasticsearch.legacy-yaml-rest-test' - -tasks.named('yamlRestTest').configure { - it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } -} +apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { description = 'A test module that allows to delay aggregations on shards with a configurable time' @@ -24,3 +20,8 @@ restResources { include '_common', 'indices', 'index', 'cluster', 'search' } } + +tasks.named('yamlRestTest') { + def isSnapshot = buildParams.snapshotBuild + it.onlyIf("snapshot build") { isSnapshot } +} diff --git a/test/external-modules/delayed-aggs/src/yamlRestTest/java/org/elasticsearch/search/aggregations/DelayedShardAggregationClientYamlTestSuiteIT.java b/test/external-modules/delayed-aggs/src/yamlRestTest/java/org/elasticsearch/search/aggregations/DelayedShardAggregationClientYamlTestSuiteIT.java index 44ea99a4cbb7d..11981646aeccd 100644 --- a/test/external-modules/delayed-aggs/src/yamlRestTest/java/org/elasticsearch/search/aggregations/DelayedShardAggregationClientYamlTestSuiteIT.java +++ b/test/external-modules/delayed-aggs/src/yamlRestTest/java/org/elasticsearch/search/aggregations/DelayedShardAggregationClientYamlTestSuiteIT.java @@ -12,10 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class DelayedShardAggregationClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("test-delayed-aggs").build(); + public DelayedShardAggregationClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -24,4 +30,9 @@ public DelayedShardAggregationClientYamlTestSuiteIT(@Name("yaml") ClientYamlTest public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/test/external-modules/die-with-dignity/build.gradle b/test/external-modules/die-with-dignity/build.gradle index 972aa0f9f0749..fd9205c05555e 100644 --- a/test/external-modules/die-with-dignity/build.gradle +++ b/test/external-modules/die-with-dignity/build.gradle @@ -17,5 +17,5 @@ tasks.named("test").configure { } tasks.named('javaRestTest').configure { - it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.snapshotBuild } } diff --git a/test/external-modules/error-query/build.gradle b/test/external-modules/error-query/build.gradle index 01e82f9168a58..17cf6f4165fe3 100644 --- a/test/external-modules/error-query/build.gradle +++ b/test/external-modules/error-query/build.gradle @@ -7,12 +7,10 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.internal.info.BuildParams -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -tasks.named('yamlRestTest').configure { - it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } -} +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { description = 'A test module that exposes a way to simulate search shard failures and warnings' @@ -24,3 +22,14 @@ restResources { include '_common', 'indices', 'index', 'cluster', 'search' } } + +dependencies { + clusterModules project(':x-pack:plugin:esql') + clusterModules project(':x-pack:plugin:autoscaling') + clusterModules project(':x-pack:plugin:ilm') +} + +tasks.withType(StandaloneRestIntegTestTask) { + def isSnapshot = buildParams.snapshotBuild + onlyIf("snapshot build") { isSnapshot } +} diff --git a/test/external-modules/error-query/src/yamlRestTest/java/org/elasticsearch/search/query/ErrorQueryClientYamlTestSuiteIT.java b/test/external-modules/error-query/src/yamlRestTest/java/org/elasticsearch/search/query/ErrorQueryClientYamlTestSuiteIT.java index bdcd10600b340..58c4326bf9031 100644 --- a/test/external-modules/error-query/src/yamlRestTest/java/org/elasticsearch/search/query/ErrorQueryClientYamlTestSuiteIT.java +++ b/test/external-modules/error-query/src/yamlRestTest/java/org/elasticsearch/search/query/ErrorQueryClientYamlTestSuiteIT.java @@ -12,10 +12,16 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class ErrorQueryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("test-error-query").build(); + public ErrorQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -24,4 +30,9 @@ public ErrorQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/test/external-modules/esql-heap-attack/build.gradle b/test/external-modules/esql-heap-attack/build.gradle index 287a82bc8e202..b7ec3d3a4b6da 100644 --- a/test/external-modules/esql-heap-attack/build.gradle +++ b/test/external-modules/esql-heap-attack/build.gradle @@ -21,5 +21,5 @@ esplugin { tasks.named('javaRestTest') { usesDefaultDistribution() - it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.snapshotBuild } } diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index f732f7cbbf00d..70f1ba529ec5e 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -89,8 +89,7 @@ public void skipOnAborted() { */ public void testSortByManyLongsSuccess() throws IOException { initManyLongs(); - Response response = sortByManyLongs(500); - Map map = responseAsMap(response); + Map response = sortByManyLongs(500); ListMatcher columns = matchesList().item(matchesMap().entry("name", "a").entry("type", "long")) .item(matchesMap().entry("name", "b").entry("type", "long")); ListMatcher values = matchesList(); @@ -99,7 +98,7 @@ public void testSortByManyLongsSuccess() throws IOException { values = values.item(List.of(0, b)); } } - assertResultMap(map, columns, values); + assertResultMap(response, columns, values); } /** @@ -107,7 +106,8 @@ public void testSortByManyLongsSuccess() throws IOException { */ public void testSortByManyLongsTooMuchMemory() throws IOException { initManyLongs(); - assertCircuitBreaks(() -> sortByManyLongs(5000)); + // 5000 is plenty to break on most nodes + assertCircuitBreaks(attempt -> sortByManyLongs(attempt * 5000)); } /** @@ -191,26 +191,42 @@ public void testSortByManyLongsTooMuchMemoryAsync() throws IOException { ); } - private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { - ResponseException e = expectThrows(ResponseException.class, r); - Map map = responseAsMap(e.getResponse()); - logger.info("expected circuit breaker {}", map); - assertMap( - map, + private static final int MAX_ATTEMPTS = 5; + + interface TryCircuitBreaking { + Map attempt(int attempt) throws IOException; + } + + private void assertCircuitBreaks(TryCircuitBreaking tryBreaking) throws IOException { + assertCircuitBreaks( + tryBreaking, matchesMap().entry("status", 429).entry("error", matchesMap().extraOk().entry("type", "circuit_breaking_exception")) ); } - private void assertFoldCircuitBreaks(ThrowingRunnable r) throws IOException { - ResponseException e = expectThrows(ResponseException.class, r); - Map map = responseAsMap(e.getResponse()); - logger.info("expected fold circuit breaking {}", map); - assertMap( - map, + private void assertFoldCircuitBreaks(TryCircuitBreaking tryBreaking) throws IOException { + assertCircuitBreaks( + tryBreaking, matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "fold_too_much_memory_exception")) ); } + private void assertCircuitBreaks(TryCircuitBreaking tryBreaking, MapMatcher responseMatcher) throws IOException { + int attempt = 1; + while (attempt <= MAX_ATTEMPTS) { + try { + Map response = tryBreaking.attempt(attempt); + logger.warn("{}: should circuit broken but got {}", attempt, response); + attempt++; + } catch (ResponseException e) { + Map map = responseAsMap(e.getResponse()); + assertMap(map, responseMatcher); + return; + } + } + fail("giving up circuit breaking after " + attempt + " attempts"); + } + private void assertParseFailure(ThrowingRunnable r) throws IOException { ResponseException e = expectThrows(ResponseException.class, r); Map map = responseAsMap(e.getResponse()); @@ -218,9 +234,9 @@ private void assertParseFailure(ThrowingRunnable r) throws IOException { assertMap(map, matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "parsing_exception"))); } - private Response sortByManyLongs(int count) throws IOException { + private Map sortByManyLongs(int count) throws IOException { logger.info("sorting by {} longs", count); - return query(makeSortByManyLongs(count).toString(), null); + return responseAsMap(query(makeSortByManyLongs(count).toString(), null)); } private StringBuilder makeSortByManyLongs(int count) { @@ -318,8 +334,7 @@ private Response concat(int evals) throws IOException { public void testManyConcat() throws IOException { int strings = 300; initManyLongs(); - Response resp = manyConcat("FROM manylongs", strings); - assertManyStrings(resp, strings); + assertManyStrings(manyConcat("FROM manylongs", strings), strings); } /** @@ -327,7 +342,8 @@ public void testManyConcat() throws IOException { */ public void testHugeManyConcat() throws IOException { initManyLongs(); - assertCircuitBreaks(() -> manyConcat("FROM manylongs", 2000)); + // 2000 is plenty to break on most nodes + assertCircuitBreaks(attempt -> manyConcat("FROM manylongs", attempt * 2000)); } /** @@ -335,18 +351,18 @@ public void testHugeManyConcat() throws IOException { */ public void testManyConcatFromRow() throws IOException { int strings = 2000; - Response resp = manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", strings); - assertManyStrings(resp, strings); + assertManyStrings(manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", strings), strings); } /** * Hits a circuit breaker by building many moderately long strings. */ public void testHugeManyConcatFromRow() throws IOException { + // 5000 is plenty to break on most nodes assertFoldCircuitBreaks( - () -> manyConcat( + attempt -> manyConcat( "ROW a=9999999999999, b=99999999999999999, c=99999999999999999, d=99999999999999999, e=99999999999999999", - 5000 + attempt * 5000 ) ); } @@ -361,7 +377,7 @@ public void testHugeHugeManyConcatFromRow() throws IOException { /** * Tests that generate many moderately long strings. */ - private Response manyConcat(String init, int strings) throws IOException { + private Map manyConcat(String init, int strings) throws IOException { StringBuilder query = startQuery(); query.append(init).append(" | EVAL str = CONCAT("); query.append( @@ -388,7 +404,7 @@ private Response manyConcat(String init, int strings) throws IOException { query.append("str").append(s); } query.append("\"}"); - return query(query.toString(), "columns"); + return responseAsMap(query(query.toString(), "columns")); } /** @@ -397,8 +413,7 @@ private Response manyConcat(String init, int strings) throws IOException { public void testManyRepeat() throws IOException { int strings = 30; initManyLongs(); - Response resp = manyRepeat("FROM manylongs", strings); - assertManyStrings(resp, 30); + assertManyStrings(manyRepeat("FROM manylongs", strings), 30); } /** @@ -406,7 +421,8 @@ public void testManyRepeat() throws IOException { */ public void testHugeManyRepeat() throws IOException { initManyLongs(); - assertCircuitBreaks(() -> manyRepeat("FROM manylongs", 75)); + // 75 is plenty to break on most nodes + assertCircuitBreaks(attempt -> manyRepeat("FROM manylongs", attempt * 75)); } /** @@ -414,15 +430,15 @@ public void testHugeManyRepeat() throws IOException { */ public void testManyRepeatFromRow() throws IOException { int strings = 300; - Response resp = manyRepeat("ROW a = 99", strings); - assertManyStrings(resp, strings); + assertManyStrings(manyRepeat("ROW a = 99", strings), strings); } /** * Hits a circuit breaker by building many moderately long strings. */ public void testHugeManyRepeatFromRow() throws IOException { - assertFoldCircuitBreaks(() -> manyRepeat("ROW a = 99", 400)); + // 400 is enough to break on most nodes + assertFoldCircuitBreaks(attempt -> manyRepeat("ROW a = 99", attempt * 400)); } /** @@ -435,7 +451,7 @@ public void testHugeHugeManyRepeatFromRow() throws IOException { /** * Tests that generate many moderately long strings. */ - private Response manyRepeat(String init, int strings) throws IOException { + private Map manyRepeat(String init, int strings) throws IOException { StringBuilder query = startQuery(); query.append(init).append(" | EVAL str = TO_STRING(a)"); for (int s = 0; s < strings; s++) { @@ -449,23 +465,21 @@ private Response manyRepeat(String init, int strings) throws IOException { query.append("str").append(s); } query.append("\"}"); - return query(query.toString(), "columns"); + return responseAsMap(query(query.toString(), "columns")); } - private void assertManyStrings(Response resp, int strings) throws IOException { - Map map = responseAsMap(resp); + private void assertManyStrings(Map resp, int strings) throws IOException { ListMatcher columns = matchesList(); for (int s = 0; s < strings; s++) { columns = columns.item(matchesMap().entry("name", "str" + s).entry("type", "keyword")); } MapMatcher mapMatcher = matchesMap(); - assertMap(map, mapMatcher.entry("columns", columns)); + assertMap(resp, mapMatcher.entry("columns", columns)); } public void testManyEval() throws IOException { initManyLongs(); - Response resp = manyEval(1); - Map map = responseAsMap(resp); + Map response = manyEval(1); ListMatcher columns = matchesList(); columns = columns.item(matchesMap().entry("name", "a").entry("type", "long")); columns = columns.item(matchesMap().entry("name", "b").entry("type", "long")); @@ -475,15 +489,16 @@ public void testManyEval() throws IOException { for (int i = 0; i < 20; i++) { columns = columns.item(matchesMap().entry("name", "i0" + i).entry("type", "long")); } - assertResultMap(map, columns, hasSize(10_000)); + assertResultMap(response, columns, hasSize(10_000)); } public void testTooManyEval() throws IOException { initManyLongs(); - assertCircuitBreaks(() -> manyEval(490)); + // 490 is plenty to fail on most nodes + assertCircuitBreaks(attempt -> manyEval(attempt * 490)); } - private Response manyEval(int evalLines) throws IOException { + private Map manyEval(int evalLines) throws IOException { StringBuilder query = startQuery(); query.append("FROM manylongs"); for (int e = 0; e < evalLines; e++) { @@ -496,7 +511,7 @@ private Response manyEval(int evalLines) throws IOException { } } query.append("\n| LIMIT 10000\"}"); - return query(query.toString(), null); + return responseAsMap(query(query.toString(), null)); } private Response query(String query, String filterPath) throws IOException { @@ -554,99 +569,161 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE public void testFetchManyBigFields() throws IOException { initManyBigFieldsIndex(100); - fetchManyBigFields(100); + Map response = fetchManyBigFields(100); + ListMatcher columns = matchesList(); + for (int f = 0; f < 1000; f++) { + columns = columns.item(matchesMap().entry("name", "f" + String.format(Locale.ROOT, "%03d", f)).entry("type", "keyword")); + } + assertMap(response, matchesMap().entry("columns", columns)); } public void testFetchTooManyBigFields() throws IOException { initManyBigFieldsIndex(500); - assertCircuitBreaks(() -> fetchManyBigFields(500)); + // 500 docs is plenty to circuit break on most nodes + assertCircuitBreaks(attempt -> fetchManyBigFields(attempt * 500)); } /** * Fetches documents containing 1000 fields which are {@code 1kb} each. */ - private void fetchManyBigFields(int docs) throws IOException { + private Map fetchManyBigFields(int docs) throws IOException { StringBuilder query = startQuery(); query.append("FROM manybigfields | SORT f000 | LIMIT " + docs + "\"}"); - Response response = query(query.toString(), "columns"); - Map map = responseAsMap(response); - ListMatcher columns = matchesList(); - for (int f = 0; f < 1000; f++) { - columns = columns.item(matchesMap().entry("name", "f" + String.format(Locale.ROOT, "%03d", f)).entry("type", "keyword")); - } - assertMap(map, matchesMap().entry("columns", columns)); + return responseAsMap(query(query.toString(), "columns")); } public void testAggMvLongs() throws IOException { int fieldValues = 100; initMvLongsIndex(1, 3, fieldValues); - Response response = aggMvLongs(3); - Map map = responseAsMap(response); + Map response = aggMvLongs(3); ListMatcher columns = matchesList().item(matchesMap().entry("name", "MAX(f00)").entry("type", "long")) .item(matchesMap().entry("name", "f00").entry("type", "long")) .item(matchesMap().entry("name", "f01").entry("type", "long")) .item(matchesMap().entry("name", "f02").entry("type", "long")); - assertMap(map, matchesMap().entry("columns", columns)); + assertMap(response, matchesMap().entry("columns", columns)); } public void testAggTooManyMvLongs() throws IOException { initMvLongsIndex(1, 3, 1000); - assertCircuitBreaks(() -> aggMvLongs(3)); + // 3 fields is plenty on most nodes + assertCircuitBreaks(attempt -> aggMvLongs(attempt * 3)); } - private Response aggMvLongs(int fields) throws IOException { + private Map aggMvLongs(int fields) throws IOException { StringBuilder query = startQuery(); query.append("FROM mv_longs | STATS MAX(f00) BY f00"); for (int f = 1; f < fields; f++) { query.append(", f").append(String.format(Locale.ROOT, "%02d", f)); } - return query(query.append("\"}").toString(), "columns"); + return responseAsMap(query(query.append("\"}").toString(), "columns")); } public void testFetchMvLongs() throws IOException { int fields = 100; initMvLongsIndex(100, fields, 1000); - Response response = fetchMvLongs(); - Map map = responseAsMap(response); + Map response = fetchMvLongs(); ListMatcher columns = matchesList(); for (int f = 0; f < fields; f++) { columns = columns.item(matchesMap().entry("name", String.format(Locale.ROOT, "f%02d", f)).entry("type", "long")); } - assertMap(map, matchesMap().entry("columns", columns)); + assertMap(response, matchesMap().entry("columns", columns)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106683") public void testFetchTooManyMvLongs() throws IOException { initMvLongsIndex(500, 100, 1000); - assertCircuitBreaks(() -> fetchMvLongs()); + assertCircuitBreaks(attempt -> fetchMvLongs()); } - private Response fetchMvLongs() throws IOException { + private Map fetchMvLongs() throws IOException { StringBuilder query = startQuery(); query.append("FROM mv_longs\"}"); - return query(query.toString(), "columns"); + return responseAsMap(query(query.toString(), "columns")); } public void testLookupExplosion() throws IOException { - int sensorDataCount = 7500; + int sensorDataCount = 500; int lookupEntries = 10000; Map map = lookupExplosion(sensorDataCount, lookupEntries); assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries)))); } public void testLookupExplosionManyMatches() throws IOException { - assertCircuitBreaks(() -> { - Map result = lookupExplosion(8500, 10000); - logger.error("should have failed but got {}", result); - }); + // 1500, 10000 is enough locally, but some CI machines need more. + assertCircuitBreaks(attempt -> lookupExplosion(attempt * 1500, 10000)); + } + + public void testLookupExplosionNoFetch() throws IOException { + int sensorDataCount = 7500; + int lookupEntries = 10000; + Map map = lookupExplosionNoFetch(sensorDataCount, lookupEntries); + assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries)))); + } + + public void testLookupExplosionNoFetchManyMatches() throws IOException { + // 8500 is plenty on most nodes + assertCircuitBreaks(attempt -> lookupExplosionNoFetch(attempt * 8500, 10000)); + } + + public void testLookupExplosionBigString() throws IOException { + int sensorDataCount = 150; + int lookupEntries = 1; + Map map = lookupExplosionBigString(sensorDataCount, lookupEntries); + assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries)))); + } + + public void testLookupExplosionBigStringManyMatches() throws IOException { + // 500, 1 is enough to make it fail locally but some CI needs more + assertCircuitBreaks(attempt -> lookupExplosionBigString(attempt * 500, 1)); + } + + private Map lookupExplosion(int sensorDataCount, int lookupEntries) throws IOException { + try { + lookupExplosionData(sensorDataCount, lookupEntries); + StringBuilder query = startQuery(); + query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(location)\"}"); + return responseAsMap(query(query.toString(), null)); + } finally { + deleteIndex("sensor_data"); + deleteIndex("sensor_lookup"); + } + } + + private Map lookupExplosionNoFetch(int sensorDataCount, int lookupEntries) throws IOException { + try { + lookupExplosionData(sensorDataCount, lookupEntries); + StringBuilder query = startQuery(); + query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(*)\"}"); + return responseAsMap(query(query.toString(), null)); + } finally { + deleteIndex("sensor_data"); + deleteIndex("sensor_lookup"); + } } - private Map lookupExplosion(int sensorDataCount, int lookupEntries) throws IOException { + private void lookupExplosionData(int sensorDataCount, int lookupEntries) throws IOException { initSensorData(sensorDataCount, 1); initSensorLookup(lookupEntries, 1, i -> "73.9857 40.7484"); - StringBuilder query = startQuery(); - query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(*)\"}"); - return responseAsMap(query(query.toString(), null)); + } + + private Map lookupExplosionBigString(int sensorDataCount, int lookupEntries) throws IOException { + try { + initSensorData(sensorDataCount, 1); + initSensorLookupString(lookupEntries, 1, i -> { + int target = Math.toIntExact(ByteSizeValue.ofMb(1).getBytes()); + StringBuilder str = new StringBuilder(Math.toIntExact(ByteSizeValue.ofMb(2).getBytes())); + while (str.length() < target) { + str.append("Lorem ipsum dolor sit amet, consectetur adipiscing elit."); + } + logger.info("big string is {} characters", str.length()); + return str.toString(); + }); + StringBuilder query = startQuery(); + query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(string)\"}"); + return responseAsMap(query(query.toString(), null)); + } finally { + deleteIndex("sensor_data"); + deleteIndex("sensor_lookup"); + } } public void testEnrichExplosion() throws IOException { @@ -657,22 +734,25 @@ public void testEnrichExplosion() throws IOException { } public void testEnrichExplosionManyMatches() throws IOException { - assertCircuitBreaks(() -> { - Map result = enrichExplosion(3000, 10000); - logger.error("should have failed but got {}", result); - }); + // 1000, 10000 is enough on most nodes + assertCircuitBreaks(attempt -> enrichExplosion(1000, attempt * 5000)); } - private Map enrichExplosion(int sensorDataCount, int lookupEntries) throws IOException { - initSensorData(sensorDataCount, 1); - initSensorEnrich(lookupEntries, 1, i -> "73.9857 40.7484"); + private Map enrichExplosion(int sensorDataCount, int lookupEntries) throws IOException { try { - StringBuilder query = startQuery(); - query.append("FROM sensor_data | ENRICH sensor ON id | STATS COUNT(*)\"}"); - return responseAsMap(query(query.toString(), null)); + initSensorData(sensorDataCount, 1); + initSensorEnrich(lookupEntries, 1, i -> "73.9857 40.7484"); + try { + StringBuilder query = startQuery(); + query.append("FROM sensor_data | ENRICH sensor ON id | STATS COUNT(*)\"}"); + return responseAsMap(query(query.toString(), null)); + } finally { + Request delete = new Request("DELETE", "/_enrich/policy/sensor"); + assertMap(responseAsMap(client().performRequest(delete)), matchesMap().entry("acknowledged", true)); + } } finally { - Request delete = new Request("DELETE", "/_enrich/policy/sensor"); - assertMap(responseAsMap(client().performRequest(delete)), matchesMap().entry("acknowledged", true)); + deleteIndex("sensor_data"); + deleteIndex("sensor_lookup"); } } @@ -834,6 +914,31 @@ private void initSensorLookup(int lookupEntries, int sensorCount, IntFunction string) throws IOException { + logger.info("loading sensor lookup with huge strings"); + createIndex("sensor_lookup", Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOOKUP.getName()).build(), """ + { + "properties": { + "id": { "type": "long" }, + "string": { "type": "text" } + } + }"""); + int docsPerBulk = 10; + StringBuilder data = new StringBuilder(); + for (int i = 0; i < lookupEntries; i++) { + int sensor = i % sensorCount; + data.append(String.format(Locale.ROOT, """ + {"create":{}} + {"id": %d, "string": "%s"} + """, sensor, string.apply(sensor))); + if (i % docsPerBulk == docsPerBulk - 1) { + bulk("sensor_lookup", data.toString()); + data.setLength(0); + } + } + initIndex("sensor_lookup", data.toString()); + } + private void initSensorEnrich(int lookupEntries, int sensorCount, IntFunction location) throws IOException { initSensorLookup(lookupEntries, sensorCount, location); logger.info("loading sensor enrich"); diff --git a/test/external-modules/jvm-crash/build.gradle b/test/external-modules/jvm-crash/build.gradle index e94bfb899e04a..5ea5be2e9755d 100644 --- a/test/external-modules/jvm-crash/build.gradle +++ b/test/external-modules/jvm-crash/build.gradle @@ -21,5 +21,5 @@ esplugin { tasks.named('javaRestTest') { usesDefaultDistribution() - it.onlyIf("snapshot build") { buildParams.isSnapshotBuild() } + it.onlyIf("snapshot build") { buildParams.snapshotBuild } } diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 5990df69f1be2..6360ead8126fa 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -10,37 +10,114 @@ apply plugin: 'elasticsearch.java' apply plugin: 'com.gradleup.shadow' + import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +def patched = Attribute.of('patched', Boolean) configurations { -// all { -// transitive = true -// } - hdfs2 - hdfs3 + hdfs2 { + attributes { + attribute(patched, true) + } + } + hdfs3 { + attributes { + attribute(patched, true) + } + } consumable("shadowedHdfs2") } dependencies { + attributesSchema { + attribute(patched) + } + artifactTypes.getByName("jar") { + attributes.attribute(patched, false) + } + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.hdfs.HdfsClassPatcher) { + from.attribute(patched, false) + to.attribute(patched, true) + parameters { + matchingArtifacts = ["hadoop-common"] + } + } + compileOnly("org.apache.hadoop:hadoop-minicluster:2.8.5") api("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}") { transitive = false } compileOnly "junit:junit:${versions.junit}" - hdfs2 "org.apache.hadoop:hadoop-minicluster:2.8.5" - hdfs3 "org.apache.hadoop:hadoop-minicluster:3.3.1" + def commonExcludes = [ + [group: "org.apache.commons", module: "commons-compress"], + [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-app"], + [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-core"], + [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-hs"], + [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-jobclient"], + [group: "org.apache.hadoop", module: "hadoop-yarn-server-tests"], + [group: "org.apache.httpcomponents", module: "httpclient"], + [group: "org.apache.zookeeper", module: "zookeeper"], + [group: "org.apache.curator", module: "curator-recipes"], + [group: "org.apache.curator", module: "curator-client"], + [group: "org.apache.curator", module: "curator-framework"], + [group: "org.apache.avro", module: "avro"], + [group: "log4j", module: "log4j"], + [group: "io.netty", module: "netty-all"], + [group: "io.netty", module: "netty"], + [group: "com.squareup.okhttp", module: "okhttp"], + [group: "com.google.guava", module: "guava"], + [group: "com.google.code.gson", module: "gson"], + [group: "javax.servlet.jsp", module: "jsp-api"], + [group: "org.fusesource.leveldbjni", module: "leveldbjni-all"], + [group: "commons-cli", module: "commons-cli"], + [group: "org.mortbay.jetty", module: "servlet-api"], + [group: "commons-logging", module: "commons-logging"], + [group: "org.slf4j", module: "slf4j-log4j12"], + [group: "commons-codec", module: "commons-codec"], + [group: "com.sun.jersey", module: "jersey-core"], + [group: "com.sun.jersey", module: "jersey-json"], + [group: "com.google.code.findbugs", module: "jsr305"], + [group: "com.sun.jersey", module: "jersey-json"], + [group: "com.nimbusds", module: "nimbus-jose-jwt"], + [group: "com.jcraft", module: "jsch"], + [group: "org.slf4j", module: "slf4j-api"], + ] + + hdfs2("org.apache.hadoop:hadoop-minicluster:2.8.5") { + commonExcludes.each { exclude it } + exclude group: "org.apache.commons", module: "commons-math3" + exclude group: "xmlenc", module: "xmlenc" + exclude group: "net.java.dev.jets3t", module: "jets3t" + exclude group: "org.apache.directory.server", module: "apacheds-i18n" + exclude group: "xerces", module: "xercesImpl" + } + + hdfs3("org.apache.hadoop:hadoop-minicluster:3.3.1") { + commonExcludes.each { exclude it } + exclude group: "dnsjava", module: "dnsjava" + exclude group: "com.google.inject.extensions", module: "guice-servlet" + exclude group: "com.google.inject", module: "guice" + exclude group: "com.microsoft.sqlserver", module: "mssql-jdbc" + exclude group: "com.sun.jersey.contribs", module: "jersey-guice" + exclude group: "com.zaxxer", module: "HikariCP-java7" + exclude group: "com.sun.jersey", module: "jersey-server" + exclude group: "org.bouncycastle", module: "bcpkix-jdk15on" + exclude group: "org.bouncycastle", module: "bcprov-jdk15on" + exclude group: "org.ehcache", module: "ehcache" + exclude group: "org.apache.geronimo.specs", module: "geronimo-jcache_1.0_spec" + exclude group: "org.xerial.snappy", module: "snappy-java" + } } tasks.named("shadowJar").configure { archiveClassifier.set("hdfs3") // fix issues with signed jars - relocate("org.apache.hadoop", "fixture.hdfs3.org.apache.hadoop") { exclude "org.apache.hadoop.hdfs.protocol.ClientProtocol" exclude "org.apache.hadoop.ipc.StandbyException" } - configurations << project.configurations.hdfs3 + configurations.add(project.configurations.hdfs3) } def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) { @@ -50,26 +127,15 @@ def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) { } archiveClassifier.set("hdfs2") from sourceSets.main.output - configurations << project.configurations.hdfs2 + configurations.add(project.configurations.hdfs2) } tasks.withType(ShadowJar).configureEach { dependencies { -// exclude(dependency('commons-io:commons-io:2.8.0')) exclude(dependency("com.carrotsearch.randomizedtesting:randomizedtesting-runner:.*")) exclude(dependency("junit:junit:.*")) - exclude(dependency("org.slf4j:slf4j-api:.*")) - exclude(dependency("com.google.guava:guava:.*")) - exclude(dependency("org.apache.commons:commons-compress:.*")) - exclude(dependency("commons-logging:commons-logging:.*")) - exclude(dependency("commons-codec:commons-codec:.*")) - exclude(dependency("org.apache.httpcomponents:httpclient:.*")) exclude(dependency("org.apache.httpcomponents:httpcore:.*")) exclude(dependency("org.apache.logging.log4j:log4j-1.2-api:.*")) - exclude(dependency("log4j:log4j:.*")) - exclude(dependency("io.netty:.*:.*")) - exclude(dependency("com.nimbusds:nimbus-jose-jwt:.*")) - exclude(dependency("commons-cli:commons-cli:1.2")) exclude(dependency("net.java.dev.jna:jna:.*")) exclude(dependency("org.objenesis:objenesis:.*")) exclude(dependency('com.fasterxml.jackson.core:.*:.*')) diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 9129fb4574ee6..7d9584491c356 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -26,9 +26,9 @@ dependencies { api "commons-codec:commons-codec:${versions.commonscodec}" // mockito - api 'org.mockito:mockito-core:5.11.0' - api 'org.mockito:mockito-subclass:5.11.0' - api 'net.bytebuddy:byte-buddy:1.14.12' + api 'org.mockito:mockito-core:5.15.2' + api 'org.mockito:mockito-subclass:5.15.2' + api 'net.bytebuddy:byte-buddy:1.15.11' api 'org.objenesis:objenesis:3.3' api "org.elasticsearch:mocksocket:${versions.mocksocket}" @@ -65,6 +65,7 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.log4j.Priority', // mockito + 'net.bytebuddy.agent.Installer', 'net.bytebuddy.agent.ByteBuddyAgent', 'org.mockito.internal.creation.bytebuddy.inject.MockMethodDispatcher', 'org.opentest4j.AssertionFailedError', diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java index b49d10ba9c402..c3384ede3a1a7 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java @@ -94,7 +94,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } public TestFileStore getTestFileStore(String nodeName) { - return fileSystemProvider.getTestFileStore(internalCluster().getInstance(Environment.class, nodeName).dataFiles()[0]); + return fileSystemProvider.getTestFileStore(internalCluster().getInstance(Environment.class, nodeName).dataDirs()[0]); } protected static class TestFileStore extends FilterFileStore { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java index c8d66f389dab1..f3db9cb50313c 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java @@ -86,6 +86,9 @@ public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return indexMetadata.getForecastedWriteLoad(); } + + @Override + public void refreshLicense() {} }; public static MockAllocationService createAllocationService() { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index 2396ea4a75f39..d54a66661b764 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -597,7 +597,7 @@ public static void getClusterStateWithDataStream( builder.put(dataStreamBuilder.build()); } - private static IndexMetadata createIndexMetadata(String name, boolean hidden, Settings settings, int replicas) { + public static IndexMetadata createIndexMetadata(String name, boolean hidden, Settings settings, int replicas) { Settings.Builder b = Settings.builder() .put(settings) .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -671,7 +671,7 @@ public static MetadataRolloverService getMetadataRolloverService( ).build(MapperBuilderContext.root(false, true)); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); - when(env.sharedDataFile()).thenReturn(null); + when(env.sharedDataDir()).thenReturn(null); AllocationService allocationService = mock(AllocationService.class); when(allocationService.reroute(any(ClusterState.class), any(String.class), any())).then(i -> i.getArguments()[0]); when(allocationService.getShardRoutingRoleStrategy()).thenReturn(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY); diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java index 0a4c99eb8b52a..74db1147f23b8 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java @@ -42,16 +42,12 @@ public class MockPluginsService extends PluginsService { * @param classpathPlugins Plugins that exist in the classpath which should be loaded */ public MockPluginsService(Settings settings, Environment environment, Collection> classpathPlugins) { - super( - settings, - environment.configFile(), - new PluginsLoader(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap()) - ); + super(settings, environment.configDir(), new PluginsLoader(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap())); List pluginsLoaded = new ArrayList<>(); for (Class pluginClass : classpathPlugins) { - Plugin plugin = loadPlugin(pluginClass, settings, environment.configFile()); + Plugin plugin = loadPlugin(pluginClass, settings, environment.configDir()); PluginDescriptor pluginInfo = new PluginDescriptor( pluginClass.getName(), "classpath plugin", diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 12094b31a049d..9e955fa727d76 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -356,6 +356,7 @@ public void testReadBlobWithPrematureConnectionClose() { containsString("premature end of chunk coded message body: closing chunk expected"), containsString("premature end of content-length delimited message body"), containsString("connection closed prematurely"), + containsString("premature eof"), // if we didn't call exchange.getResponseBody().flush() then we might not even have sent the response headers: alwaysFlushBody ? never() : containsString("the target server failed to respond") ) diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java index 6ae95b872a75f..f3861e9279ef2 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java @@ -91,7 +91,7 @@ public static void startHttpServer() throws Exception { // the EncryptedRepository can require more than one connection open at one time executorService = EsExecutors.newScaling( ESMockAPIBasedRepositoryIntegTestCase.class.getName(), - 0, + 1, 2, 60, TimeUnit.SECONDS, diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 1e82313338b97..24d46b99b541b 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -137,6 +137,12 @@ public double execute() { Map vars = new HashMap<>(parameters); vars.put("params", parameters); vars.put("doc", getDoc()); + try { + vars.put("_score", get_score()); + } catch (Exception ignore) { + // nothing to do: if get_score throws we don't set the _score, likely the scorer is null, + // which is ok if _score was not requested e.g. top_hits. + } return ((Number) script.apply(vars)).doubleValue(); } }; @@ -881,6 +887,12 @@ public String execute() { Map vars = new HashMap<>(parameters); vars.put("params", parameters); vars.put("doc", getDoc()); + try { + vars.put("_score", get_score()); + } catch (Exception ignore) { + // nothing to do: if get_score throws we don't set the _score, likely the scorer is null, + // which is ok if _score was not requested e.g. top_hits. + } return String.valueOf(script.apply(vars)); } }; @@ -907,6 +919,12 @@ public BytesRefProducer execute() { Map vars = new HashMap<>(parameters); vars.put("params", parameters); vars.put("doc", getDoc()); + try { + vars.put("_score", get_score()); + } catch (Exception ignore) { + // nothing to do: if get_score throws we don't set the _score, likely the scorer is null, + // which is ok if _score was not requested e.g. top_hits. + } return (BytesRefProducer) script.apply(vars); } }; diff --git a/test/framework/src/main/java/org/elasticsearch/search/ErrorTraceHelper.java b/test/framework/src/main/java/org/elasticsearch/search/ErrorTraceHelper.java new file mode 100644 index 0000000000000..5912c04ffb22f --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/search/ErrorTraceHelper.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search; + +import org.apache.logging.log4j.Level; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.MockLog; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.Strings.format; +import static org.elasticsearch.test.ESIntegTestCase.getNodeId; +import static org.elasticsearch.test.ESIntegTestCase.internalCluster; + +/** + * Utilities around testing the `error_trace` message header in search. + */ +public enum ErrorTraceHelper { + ; + + /** + * Adds expectations for debug logging of a message and exception on each shard of the given index. + * + * @param numShards the number of shards in the index (an expectation will be added for each shard) + * @param mockLog the mock log + * @param errorTriggeringIndex the name of the index that will trigger the error + */ + public static void addSeenLoggingExpectations(int numShards, MockLog mockLog, String errorTriggeringIndex) { + String nodesDisjunction = format( + "(%s)", + Arrays.stream(internalCluster().getNodeNames()).map(ESIntegTestCase::getNodeId).collect(Collectors.joining("|")) + ); + for (int shard = 0; shard < numShards; shard++) { + mockLog.addExpectation( + new MockLog.PatternAndExceptionSeenEventExpectation( + format( + "\"[%s][%s][%d]: failed to execute search request for task [\\d+]\" and an exception logged", + nodesDisjunction, + errorTriggeringIndex, + shard + ), + SearchService.class.getCanonicalName(), + Level.DEBUG, + format( + "\\[%s\\]\\[%s\\]\\[%d\\]: failed to execute search request for task \\[\\d+\\]", + nodesDisjunction, + errorTriggeringIndex, + shard + ), + QueryShardException.class, + "failed to create query: For input string: \"foo\"" + ) + ); + } + } + + /** + * Adds expectations for the _absence_ of debug logging of a message. An unseen expectation is added for each + * combination of node in the internal cluster and shard in the index. + * + * @param numShards the number of shards in the index (an expectation will be added for each shard) + * @param mockLog the mock log + * @param errorTriggeringIndex the name of the index that will trigger the error + */ + public static void addUnseenLoggingExpectations(int numShards, MockLog mockLog, String errorTriggeringIndex) { + for (String nodeName : internalCluster().getNodeNames()) { + for (int shard = 0; shard < numShards; shard++) { + mockLog.addExpectation( + new MockLog.UnseenEventExpectation( + format( + "\"[%s][%s][%d]: failed to execute search request\" and an exception logged", + getNodeId(nodeName), + errorTriggeringIndex, + shard + ), + SearchService.class.getCanonicalName(), + Level.DEBUG, + format("[%s][%s][%d]: failed to execute search request", getNodeId(nodeName), errorTriggeringIndex, shard) + ) + ); + } + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 366dbe05cf7f1..392a815946a33 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -127,6 +127,7 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MetricsAggregator; import org.elasticsearch.search.aggregations.metrics.MultiValueAggregation; @@ -147,6 +148,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.internal.SubSearchContext; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -248,29 +250,12 @@ protected List getSearchPlugins() { return List.of(); } - /** - * Deprecated - this will be made private in a future update - */ - @Deprecated - protected A createAggregator( - AggregationBuilder aggregationBuilder, - IndexReader indexReader, - MappedFieldType... fieldTypes - ) throws IOException { - return createAggregator(aggregationBuilder, createAggregationContext(indexReader, new MatchAllDocsQuery(), fieldTypes)); - } - protected A createAggregator(AggregationBuilder aggregationBuilder, AggregationContext context) throws IOException { return createAggregator(new AggregatorFactories.Builder().addAggregator(aggregationBuilder), context); } - /** - * Deprecated - this will be made private in a future update - */ - @Deprecated - protected A createAggregator(AggregatorFactories.Builder builder, AggregationContext context) - throws IOException { + private A createAggregator(AggregatorFactories.Builder builder, AggregationContext context) throws IOException { Aggregator[] aggregators = builder.build(context, null).createTopLevelAggregators(); assertThat(aggregators.length, equalTo(1)); @SuppressWarnings("unchecked") @@ -307,10 +292,7 @@ protected AggregationContext createAggregationContext(IndexReader indexReader, Q * While {@linkplain AggregationContext} is {@link Releasable} the caller is * not responsible for releasing it. Instead, it is released automatically in * in {@link #cleanupReleasables()}. - * - * Deprecated - this will be made private in a future update */ - @Deprecated protected AggregationContext createAggregationContext( IndexReader indexReader, IndexSettings indexSettings, @@ -343,6 +325,56 @@ private AggregationContext createAggregationContext( int maxBucket, boolean isInSortOrderExecutionRequired, MappedFieldType... fieldTypes + ) { + return createAggregationContext( + searcher, + indexSettings, + query, + breakerService, + bytesToPreallocate, + maxBucket, + isInSortOrderExecutionRequired, + () -> false, + fieldTypes + ); + } + + /** + * Creates an aggregation context that will randomly report that the query has been cancelled + */ + private AggregationContext createCancellingAggregationContext( + IndexSearcher searcher, + IndexSettings indexSettings, + Query query, + CircuitBreakerService breakerService, + long bytesToPreallocate, + int maxBucket, + boolean isInSortOrderExecutionRequired, + MappedFieldType... fieldTypes + ) { + return createAggregationContext( + searcher, + indexSettings, + query, + breakerService, + bytesToPreallocate, + maxBucket, + isInSortOrderExecutionRequired, + () -> ESTestCase.random().nextInt(20) == 0, + fieldTypes + ); + } + + private AggregationContext createAggregationContext( + IndexSearcher searcher, + IndexSettings indexSettings, + Query query, + CircuitBreakerService breakerService, + long bytesToPreallocate, + int maxBucket, + boolean isInSortOrderExecutionRequired, + Supplier isCancelled, + MappedFieldType... fieldTypes ) { MappingLookup mappingLookup = MappingLookup.fromMappers( Mapping.EMPTY, @@ -406,7 +438,7 @@ public Iterable dimensionFields() { bitsetFilterCache, randomInt(), () -> 0L, - () -> false, + isCancelled, q -> q, true, isInSortOrderExecutionRequired @@ -533,9 +565,11 @@ protected A searchAndReduce(IndexReader reader, IndexSettings indexSettings = createIndexSettings(); // First run it to find circuit breaker leaks on the aggregator runWithCrankyCircuitBreaker(indexSettings, searcher, aggTestConfig); - // Second run it to the end CircuitBreakerService breakerService = new NoneCircuitBreakerService(); - return searchAndReduce(indexSettings, searcher, breakerService, aggTestConfig); + // Next, try with random cancellations, again looking for leaks + runWithCancellingConfig(indexSettings, searcher, breakerService, aggTestConfig); + // Finally, run it to the end + return searchAndReduce(indexSettings, searcher, breakerService, aggTestConfig, this::createAggregationContext); } /** @@ -549,7 +583,7 @@ private void runWithCrankyCircuitBreaker(IndexSettings indexSettings, IndexSearc CircuitBreakerService crankyService = new CrankyCircuitBreakerService(); for (int i = 0; i < 5; i++) { try { - searchAndReduce(indexSettings, searcher, crankyService, aggTestConfig); + searchAndReduce(indexSettings, searcher, crankyService, aggTestConfig, this::createAggregationContext); } catch (CircuitBreakingException e) { // Circuit breaks from the cranky breaker are expected - it randomly fails, after all assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); @@ -557,12 +591,43 @@ private void runWithCrankyCircuitBreaker(IndexSettings indexSettings, IndexSearc } } + private void runWithCancellingConfig( + IndexSettings indexSettings, + IndexSearcher searcher, + CircuitBreakerService breakerService, + AggTestConfig aggTestConfig + ) throws IOException { + for (int i = 0; i < 5; i++) { + try { + searchAndReduce(indexSettings, searcher, breakerService, aggTestConfig, this::createCancellingAggregationContext); + } catch (TaskCancelledException e) { + // we don't want to expectThrows this because the randomizer might just never report cancellation, + // but it's also normal that it should throw here. + } + } + } + + @FunctionalInterface + public interface AggregationcContextSupplier { + AggregationContext get( + IndexSearcher searcher, + IndexSettings indexSettings, + Query query, + CircuitBreakerService breakerService, + long bytesToPreallocate, + int maxBucket, + boolean isInSortOrderExecutionRequired, + MappedFieldType... fieldTypes + ); + } + @SuppressWarnings("unchecked") private A searchAndReduce( IndexSettings indexSettings, IndexSearcher searcher, CircuitBreakerService breakerService, - AggTestConfig aggTestConfig + AggTestConfig aggTestConfig, + AggregationcContextSupplier contextSupplier ) throws IOException { Query query = aggTestConfig.query(); AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(aggTestConfig.builder()); @@ -588,7 +653,7 @@ private A searchAndReduce( subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX); } for (ShardSearcher subSearcher : subSearchers) { - AggregationContext context = createAggregationContext( + AggregationContext context = contextSupplier.get( subSearcher, indexSettings, query, @@ -617,7 +682,7 @@ private A searchAndReduce( } } } else { - AggregationContext context = createAggregationContext( + AggregationContext context = contextSupplier.get( searcher, indexSettings, query, @@ -687,25 +752,27 @@ private A searchAndReduce( assertRoundTrip(internalAggregation.copyResults()); } } + /* Verify that cancellation during final reduce correctly throws. + * We check reduce time cancellation only when consuming buckets. + */ + if (aggTestConfig.testReductionCancellation()) { + try { + // I can't remember if we mutate the InternalAggregations list, so make a defensive copy + List internalAggsCopy = new ArrayList<>(internalAggs); + A internalAgg = doFinalReduce(maxBucket, bigArraysForReduction, builder, internalAggsCopy, true); + if (internalAgg instanceof MultiBucketsAggregation mb) { + // Empty mutli-bucket aggs are expected to return before even getting to the cancellation check + assertEquals("Got non-empty result for a cancelled reduction", 0, mb.getBuckets().size()); + } // other cases? + } catch (TaskCancelledException e) { + /* We may not always honor cancellation in reduce, for example if we are returning no results, so we can't + * just expectThrows here. + */ + } + } // now do the final reduce - MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer( - maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) - ); - AggregationReduceContext reduceContext = new AggregationReduceContext.ForFinal( - bigArraysForReduction, - getMockScriptService(), - () -> false, - builder, - reduceBucketConsumer - ); - - @SuppressWarnings("unchecked") - A internalAgg = (A) doInternalAggregationsReduce(internalAggs, reduceContext); - assertRoundTrip(internalAgg); - - doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); + A internalAgg = doFinalReduce(maxBucket, bigArraysForReduction, builder, internalAggs, false); assertRoundTrip(internalAgg); if (aggTestConfig.builder instanceof ValuesSourceAggregationBuilder.MetricsAggregationBuilder) { verifyMetricNames((ValuesSourceAggregationBuilder.MetricsAggregationBuilder) aggTestConfig.builder, internalAgg); @@ -716,6 +783,34 @@ private A searchAndReduce( } } + private A doFinalReduce( + int maxBucket, + BigArrays bigArraysForReduction, + Builder builder, + List internalAggs, + boolean cancelled + ) throws IOException { + MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer( + maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) + ); + AggregationReduceContext reduceContext = new AggregationReduceContext.ForFinal( + bigArraysForReduction, + getMockScriptService(), + () -> cancelled, + builder, + reduceBucketConsumer + ); + + @SuppressWarnings("unchecked") + A internalAgg = (A) doInternalAggregationsReduce(internalAggs, reduceContext); + assertRoundTrip(internalAgg); + + doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); + + return internalAgg; + } + private InternalAggregation doReduce(List aggregators, AggregationReduceContext reduceContext) { final List internalAggregations = new ArrayList<>(aggregators.size()); for (InternalAggregation aggregator : aggregators) { @@ -1591,11 +1686,12 @@ public record AggTestConfig( boolean incrementalReduce, boolean useLogDocMergePolicy, + boolean testReductionCancellation, MappedFieldType... fieldTypes ) { public AggTestConfig(AggregationBuilder builder, MappedFieldType... fieldTypes) { - this(new MatchAllDocsQuery(), builder, DEFAULT_MAX_BUCKETS, randomBoolean(), true, randomBoolean(), false, fieldTypes); + this(new MatchAllDocsQuery(), builder, DEFAULT_MAX_BUCKETS, randomBoolean(), true, randomBoolean(), false, true, fieldTypes); } public AggTestConfig withQuery(Query query) { @@ -1607,6 +1703,7 @@ public AggTestConfig withQuery(Query query) { shouldBeCached, incrementalReduce, useLogDocMergePolicy, + testReductionCancellation, fieldTypes ); } @@ -1620,6 +1717,7 @@ public AggTestConfig withSplitLeavesIntoSeperateAggregators(boolean splitLeavesI shouldBeCached, incrementalReduce, useLogDocMergePolicy, + testReductionCancellation, fieldTypes ); } @@ -1633,6 +1731,7 @@ public AggTestConfig withShouldBeCached(boolean shouldBeCached) { shouldBeCached, incrementalReduce, useLogDocMergePolicy, + testReductionCancellation, fieldTypes ); } @@ -1646,6 +1745,7 @@ public AggTestConfig withMaxBuckets(int maxBuckets) { shouldBeCached, incrementalReduce, useLogDocMergePolicy, + testReductionCancellation, fieldTypes ); } @@ -1659,6 +1759,7 @@ public AggTestConfig withIncrementalReduce(boolean incrementalReduce) { shouldBeCached, incrementalReduce, useLogDocMergePolicy, + testReductionCancellation, fieldTypes ); } @@ -1672,6 +1773,21 @@ public AggTestConfig withLogDocMergePolicy() { shouldBeCached, incrementalReduce, true, + testReductionCancellation, + fieldTypes + ); + } + + public AggTestConfig noReductionCancellation() { + return new AggTestConfig( + query, + builder, + maxBuckets, + splitLeavesIntoSeparateAggregators, + shouldBeCached, + incrementalReduce, + useLogDocMergePolicy, + false, fieldTypes ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java index cc35f63d289eb..39b0f2b60662e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java @@ -145,8 +145,21 @@ private XContentTester( public void test() throws IOException { for (int runs = 0; runs < numberOfTestRuns; runs++) { XContentType xContentType = randomFrom(XContentType.values()).canonical(); - T testInstance = instanceSupplier.apply(xContentType); + T testInstance = null; try { + if (xContentType.equals(XContentType.YAML)) { + testInstance = randomValueOtherThanMany(instance -> { + // unicode character U+0085 (NEXT LINE (NEL)) doesn't survive YAML round trip tests (see #97716) + // get a new random instance if we detect this character in the xContent output + try { + return toXContent.apply(instance, xContentType).utf8ToString().contains("\u0085"); + } catch (IOException e) { + throw new AssertionError(e); + } + }, () -> instanceSupplier.apply(xContentType)); + } else { + testInstance = instanceSupplier.apply(xContentType); + } BytesReference originalXContent = toXContent.apply(testInstance, xContentType); BytesReference shuffledContent = insertRandomFieldsAndShuffle( originalXContent, @@ -173,7 +186,9 @@ public void test() throws IOException { dispose.accept(parsed); } } finally { - dispose.accept(testInstance); + if (testInstance != null) { + dispose.accept(testInstance); + } } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 2eef184718ee3..38f69a23b6855 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -48,6 +48,7 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchRequest; @@ -74,6 +75,7 @@ import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -576,10 +578,10 @@ private void afterInternal(boolean afterClass) throws Exception { ensureClusterInfoServiceRunning(); beforeIndexDeletion(); cluster().wipe(excludeTemplates()); // wipe after to make sure we fail in the test that didn't ack the delete + cluster().assertAfterTest(); if (afterClass || currentClusterScope == Scope.TEST) { cluster().close(); } - cluster().assertAfterTest(); } } finally { if (currentClusterScope == Scope.TEST) { @@ -837,6 +839,22 @@ private static Settings.Builder getExcludeSettings(int num, Settings.Builder bui return builder; } + /** + * Returns a list of the data stream's backing index names. + */ + public List getDataStreamBackingIndexNames(String dataStreamName) { + GetDataStreamAction.Response response = safeGet( + client().execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }) + ) + ); + assertThat(response.getDataStreams().size(), equalTo(1)); + DataStream dataStream = response.getDataStreams().get(0).getDataStream(); + assertThat(dataStream.getName(), equalTo(dataStreamName)); + return dataStream.getIndices().stream().map(Index::getName).toList(); + } + /** * Waits until all nodes have no pending tasks. */ @@ -2221,7 +2239,7 @@ public static Path randomRepoPath() { */ public static Path randomRepoPath(Settings settings) { Environment environment = TestEnvironment.newEnvironment(settings); - Path[] repoFiles = environment.repoFiles(); + Path[] repoFiles = environment.repoDirs(); assert repoFiles.length > 0; Path path; do { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 5b4890e995838..f6dcfaa6ab3cc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -117,6 +117,7 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.jdk.RuntimeVersionFeature; +import org.elasticsearch.logging.internal.spi.LoggerFactory; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; @@ -576,6 +577,23 @@ public void removeHeaderWarningAppender() { } } + private static org.elasticsearch.logging.Level capturedLogLevel = null; + + // just capture the expected level once before the suite starts + @BeforeClass + public static void captureLoggingLevel() { + capturedLogLevel = LoggerFactory.provider().getRootLevel(); + } + + @AfterClass + public static void restoreLoggingLevel() { + if (capturedLogLevel != null) { + // log level might not have been captured if suite was skipped + LoggerFactory.provider().setRootLevel(capturedLogLevel); + capturedLogLevel = null; + } + } + @Before public final void before() { LeakTracker.setContextHint(getTestName()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 51dca41ab5446..a5407c7690e52 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1803,7 +1803,7 @@ private void rebuildUnicastHostFiles(List newNodes) { .distinct() .collect(Collectors.toList()); Set configPaths = Stream.concat(currentNodes.stream(), newNodes.stream()) - .map(nac -> nac.node.getEnvironment().configFile()) + .map(nac -> nac.node.getEnvironment().configDir()) .collect(Collectors.toSet()); logger.debug("configuring discovery with {} at {}", discoveryFileContents, configPaths); for (final Path configPath : configPaths) { @@ -1817,7 +1817,7 @@ private void rebuildUnicastHostFiles(List newNodes) { } public Collection configPaths() { - return nodes.values().stream().map(nac -> nac.node.getEnvironment().configFile()).toList(); + return nodes.values().stream().map(nac -> nac.node.getEnvironment().configDir()).toList(); } private void stopNodesAndClient(NodeAndClient nodeAndClient) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/test/LambdaMatchers.java b/test/framework/src/main/java/org/elasticsearch/test/LambdaMatchers.java index dd2e8e4ec5506..a329b5fbaebb2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/LambdaMatchers.java +++ b/test/framework/src/main/java/org/elasticsearch/test/LambdaMatchers.java @@ -23,58 +23,63 @@ public class LambdaMatchers { - private static class TransformMatcher extends BaseMatcher { + private static class TransformMatcher extends TypeSafeMatcher { + private final String transformDescription; private final Matcher matcher; private final Function transform; - private TransformMatcher(Matcher matcher, Function transform) { + private TransformMatcher(String transformDescription, Matcher matcher, Function transform) { + this.transformDescription = transformDescription; this.matcher = matcher; this.transform = transform; } @Override - @SuppressWarnings("unchecked") - public boolean matches(Object actual) { + protected boolean matchesSafely(T item) { U u; try { - u = transform.apply((T) actual); + u = transform.apply(item); } catch (ClassCastException e) { throw new AssertionError(e); } - return matcher.matches(u); } @Override - @SuppressWarnings("unchecked") - public void describeMismatch(Object item, Description description) { + protected void describeMismatchSafely(T item, Description description) { U u; try { - u = transform.apply((T) item); + u = transform.apply(item); } catch (ClassCastException e) { description.appendValue(item).appendText(" is not of the correct type (").appendText(e.getMessage()).appendText(")"); return; } - description.appendText("transformed value "); + description.appendText(transformDescription).appendText(" "); matcher.describeMismatch(u, description); } @Override public void describeTo(Description description) { - description.appendText("transformed to match ").appendDescriptionOf(matcher); + description.appendText(transformDescription).appendText(" matches ").appendDescriptionOf(matcher); } } public static Matcher transformedMatch(Function function, Matcher matcher) { - return new TransformMatcher<>(matcher, function); + return new TransformMatcher<>("transformed value", matcher, function); + } + + public static Matcher transformedMatch(String description, Function function, Matcher matcher) { + return new TransformMatcher<>(description, matcher, function); } private static class ListTransformMatcher extends TypeSafeMatcher> { + private final String transformDescription; private final Matcher> matcher; private final Function transform; - private ListTransformMatcher(Matcher> matcher, Function transform) { + private ListTransformMatcher(String transformDescription, Matcher> matcher, Function transform) { + this.transformDescription = transformDescription; this.matcher = matcher; this.transform = transform; } @@ -110,25 +115,35 @@ protected void describeMismatchSafely(Iterable item, Description description) } } - description.appendText("transformed item "); + description.appendText(transformDescription).appendText(" "); matcher.describeMismatch(us, description); } @Override public void describeTo(Description description) { - description.appendText("iterable with transformed items to match ").appendDescriptionOf(matcher); + description.appendText("iterable with ").appendText(transformDescription).appendText(" matching ").appendDescriptionOf(matcher); } } public static Matcher> transformedItemsMatch(Function function, Matcher> matcher) { - return new ListTransformMatcher<>(matcher, function); + return new ListTransformMatcher<>("transformed items", matcher, function); + } + + public static Matcher> transformedItemsMatch( + String transformDescription, + Function function, + Matcher> matcher + ) { + return new ListTransformMatcher<>(transformDescription, matcher, function); } private static class ArrayTransformMatcher extends TypeSafeMatcher { + private final String transformDescription; private final Matcher matcher; private final Function transform; - private ArrayTransformMatcher(Matcher matcher, Function transform) { + private ArrayTransformMatcher(String transformDescription, Matcher matcher, Function transform) { + this.transformDescription = transformDescription; this.matcher = matcher; this.transform = transform; } @@ -177,18 +192,26 @@ protected void describeMismatchSafely(T[] item, Description description) { us[i] = u; } - description.appendText("transformed item "); + description.appendText(transformDescription).appendText(" "); matcher.describeMismatch(us, description); } @Override public void describeTo(Description description) { - description.appendText("array with transformed items to match ").appendDescriptionOf(matcher); + description.appendText("array with ").appendText(transformDescription).appendText(" matching ").appendDescriptionOf(matcher); } } public static Matcher transformedArrayItemsMatch(Function function, Matcher matcher) { - return new ArrayTransformMatcher<>(matcher, function); + return new ArrayTransformMatcher<>("transformed items", matcher, function); + } + + public static Matcher transformedArrayItemsMatch( + String transformDescription, + Function function, + Matcher matcher + ) { + return new ArrayTransformMatcher<>(transformDescription, matcher, function); } private static class PredicateMatcher extends BaseMatcher> { diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLog.java b/test/framework/src/main/java/org/elasticsearch/test/MockLog.java index 4a012bb361e65..4eb9ce7f4a722 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockLog.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockLog.java @@ -316,6 +316,41 @@ public void awaitMatched(long millis) throws InterruptedException { } } + public static class PatternAndExceptionSeenEventExpectation extends SeenEventExpectation { + + private final Pattern pattern; + private final Class clazz; + private final String exceptionMessage; + + public PatternAndExceptionSeenEventExpectation( + String name, + String logger, + Level level, + String pattern, + Class clazz, + String exceptionMessage + ) { + super(name, logger, level, pattern); + this.pattern = Pattern.compile(pattern); + this.clazz = clazz; + this.exceptionMessage = exceptionMessage; + } + + @Override + public void match(LogEvent event) { + if (event.getLevel().equals(level) && event.getLoggerName().equals(logger)) { + boolean patternMatches = pattern.matcher(event.getMessage().getFormattedMessage()).matches(); + boolean exceptionMatches = event.getThrown() != null + && event.getThrown().getClass() == clazz + && event.getThrown().getMessage().equals(exceptionMessage); + + if (patternMatches && exceptionMatches) { + seenLatch.countDown(); + } + } + } + } + /** * A wrapper around {@link LoggingExpectation} to detect if the assertMatched method has been called */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java b/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java index 47eeaab5dca62..2998ac8470fe7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java +++ b/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java @@ -175,6 +175,10 @@ public String getAddress() { return "http://127.0.0.1:" + httpServer.getAddress().getPort(); } + public String getHostAndPort() { + return "127.0.0.1:" + httpServer.getAddress().getPort(); + } + @FunctionalInterface public interface RequestHandler { Response handle(Request request) throws IOException; diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index c5bcf2c97d385..dc50ea40567a7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -255,7 +255,7 @@ public static void assertBlocked(@Nullable final Integer expectedBlockId, Except assertThat( "Request should have been blocked by [" + expectedBlockId + "] instead of " + e.blocks(), e.blocks(), - hasItem(transformedMatch(ClusterBlock::id, equalTo(expectedBlockId))) + hasItem(transformedMatch("ClusterBlock id", ClusterBlock::id, equalTo(expectedBlockId))) ); } } @@ -762,33 +762,39 @@ public static void assertSuggestion(Suggest searchSuggest, int entry, String key * Assert that an index template is missing */ public static void assertIndexTemplateMissing(GetIndexTemplatesResponse templatesResponse, String name) { - assertThat(templatesResponse.getIndexTemplates(), not(hasItem(transformedMatch(IndexTemplateMetadata::name, equalTo(name))))); + assertThat( + templatesResponse.getIndexTemplates(), + not(hasItem(transformedMatch("IndexTemplateMetadata name", IndexTemplateMetadata::name, equalTo(name)))) + ); } /** * Assert that an index template exists */ public static void assertIndexTemplateExists(GetIndexTemplatesResponse templatesResponse, String name) { - assertThat(templatesResponse.getIndexTemplates(), hasItem(transformedMatch(IndexTemplateMetadata::name, equalTo(name)))); + assertThat( + templatesResponse.getIndexTemplates(), + hasItem(transformedMatch("IndexTemplateMetadata name", IndexTemplateMetadata::name, equalTo(name))) + ); } /* * matchers */ public static Matcher hasId(final String id) { - return transformedMatch(SearchHit::getId, equalTo(id)); + return transformedMatch("SearchHit id", SearchHit::getId, equalTo(id)); } public static Matcher hasIndex(final String index) { - return transformedMatch(SearchHit::getIndex, equalTo(index)); + return transformedMatch("SearchHit index", SearchHit::getIndex, equalTo(index)); } public static Matcher hasScore(final float score) { - return transformedMatch(SearchHit::getScore, equalTo(score)); + return transformedMatch("SearchHit score", SearchHit::getScore, equalTo(score)); } public static Matcher hasRank(final int rank) { - return transformedMatch(SearchHit::getRank, equalTo(rank)); + return transformedMatch("SearchHit rank", SearchHit::getRank, equalTo(rank)); } public static T assertBooleanSubQuery(Query query, Class subqueryType, int i) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 48860210da06f..7e2725a27ebe5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2069,6 +2069,20 @@ protected static boolean aliasExists(String index, String alias) throws IOExcept return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); } + /** + * Returns a list of the data stream's backing index names. + */ + @SuppressWarnings("unchecked") + protected static List getDataStreamBackingIndexNames(String dataStreamName) throws IOException { + Map response = getAsMap(client(), "/_data_stream/" + dataStreamName); + List dataStreams = (List) response.get("data_streams"); + assertThat(dataStreams.size(), equalTo(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(dataStreamName)); + List indices = (List) dataStream.get("indices"); + return indices.stream().map(index -> ((Map) index).get("index_name")).toList(); + } + @SuppressWarnings("unchecked") protected static Map getAlias(final String index, final String alias) throws IOException { String endpoint = "/_alias"; diff --git a/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java index b8f4dcb399ec7..e3cc3bba94a5c 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java @@ -12,11 +12,13 @@ import com.carrotsearch.randomizedtesting.RandomizedContext; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; +import java.io.IOException; import java.util.Map; import static org.hamcrest.Matchers.equalTo; @@ -49,4 +51,42 @@ public void testInsertRandomFieldsAndShuffle() throws Exception { assertThat(mapOrdered.keySet().iterator().next(), not(equalTo("field"))); } } + + private record TestToXContent(String field, String value) implements ToXContentFragment { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field(field, value); + } + } + + public void testYamlXContentRoundtripSanitization() throws Exception { + var test = new AbstractXContentTestCase() { + + @Override + protected TestToXContent createTestInstance() { + // we need to randomly create both a "problematic" and an okay version in order to ensure that the sanitization code + // can draw at least one okay version if polled often enough + return randomBoolean() ? new TestToXContent("a\u0085b", "def") : new TestToXContent("a b", "def"); + } + + @Override + protected TestToXContent doParseInstance(XContentParser parser) throws IOException { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + String name = parser.currentName(); + assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); + String value = parser.text(); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + return new TestToXContent(name, value); + }; + + @Override + protected boolean supportsUnknownFields() { + return false; + } + }; + // testFromXContent runs 20 repetitions, enough to hit a YAML xcontent version very likely + test.testFromXContent(); + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/LambdaMatchersTests.java b/test/framework/src/test/java/org/elasticsearch/test/LambdaMatchersTests.java index f7ab2349ec1ce..c97a369a9853e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/LambdaMatchersTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/LambdaMatchersTests.java @@ -19,11 +19,13 @@ import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; import static org.elasticsearch.test.LambdaMatchers.transformedMatch; import static org.elasticsearch.test.LambdaMatchers.trueWith; +import static org.hamcrest.Matchers.anything; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; public class LambdaMatchersTests extends ESTestCase { @@ -56,11 +58,13 @@ public void testTransformMatcher() { assertThat(new A("1"), transformedMatch(a -> a.str, equalTo("1"))); assertThat(new B("1"), transformedMatch((A a) -> a.str, equalTo("1"))); + assertMismatch((A) null, transformedMatch(A::toString, anything()), is("was null")); assertMismatch(new A("1"), transformedMatch(a -> a.str, emptyString()), equalTo("transformed value was \"1\"")); } public void testTransformDescription() { - assertDescribeTo(transformedMatch((A a) -> a.str, emptyString()), equalTo("transformed to match an empty string")); + assertDescribeTo(transformedMatch((A a) -> a.str, emptyString()), equalTo("transformed value matches an empty string")); + assertDescribeTo(transformedMatch("str field", (A a) -> a.str, emptyString()), equalTo("str field matches an empty string")); } public void testListTransformMatcher() { @@ -71,14 +75,23 @@ public void testListTransformMatcher() { assertMismatch( as, transformedItemsMatch(a -> a.str, containsInAnyOrder("1", "2", "4")), - equalTo("transformed item not matched: \"3\"") + equalTo("transformed items not matched: \"3\"") + ); + assertMismatch( + as, + transformedItemsMatch("str field", a -> a.str, containsInAnyOrder("1", "2", "4")), + equalTo("str field not matched: \"3\"") ); } public void testListTransformDescription() { assertDescribeTo( transformedItemsMatch((A a) -> a.str, containsInAnyOrder("1")), - equalTo("iterable with transformed items to match iterable with items [\"1\"] in any order") + equalTo("iterable with transformed items matching iterable with items [\"1\"] in any order") + ); + assertDescribeTo( + transformedItemsMatch("str field", (A a) -> a.str, containsInAnyOrder("1")), + equalTo("iterable with str field matching iterable with items [\"1\"] in any order") ); } @@ -89,14 +102,23 @@ public void testArrayTransformMatcher() { assertMismatch( as, transformedArrayItemsMatch(a -> a.str, arrayContainingInAnyOrder("1", "2", "4")), - equalTo("transformed item not matched: \"3\"") + equalTo("transformed items not matched: \"3\"") + ); + assertMismatch( + as, + transformedArrayItemsMatch("str field", a -> a.str, arrayContainingInAnyOrder("1", "2", "4")), + equalTo("str field not matched: \"3\"") ); } public void testArrayTransformDescription() { assertDescribeTo( transformedArrayItemsMatch((A a) -> a.str, arrayContainingInAnyOrder("1")), - equalTo("array with transformed items to match [\"1\"] in any order") + equalTo("array with transformed items matching [\"1\"] in any order") + ); + assertDescribeTo( + transformedArrayItemsMatch("str field", (A a) -> a.str, arrayContainingInAnyOrder("1")), + equalTo("array with str field matching [\"1\"] in any order") ); } diff --git a/test/test-clusters/build.gradle b/test/test-clusters/build.gradle index d2c7633603f26..da87b75c2d588 100644 --- a/test/test-clusters/build.gradle +++ b/test/test-clusters/build.gradle @@ -9,6 +9,10 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" implementation "org.elasticsearch.gradle:reaper" + + testImplementation "junit:junit:${versions.junit}" + testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" + testImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" } tasks.named("processResources").configure { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSystemPropertyProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSystemPropertyProvider.java new file mode 100644 index 0000000000000..0b81eab4cffb7 --- /dev/null +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSystemPropertyProvider.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.test.cluster; + +import org.elasticsearch.test.cluster.local.LocalClusterSpec; + +import java.util.HashMap; +import java.util.Map; + +public class MutableSystemPropertyProvider implements SystemPropertyProvider { + private final Map settings = new HashMap<>(); + + @Override + public Map get(LocalClusterSpec.LocalNodeSpec nodeSpec) { + return settings; + } + + public void put(String setting, String value) { + settings.put(setting, value); + } + + public void remove(String setting) { + settings.remove(setting); + } + + public void clear() { + settings.clear(); + } +} diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index cfdca56542eb2..50e92ef718eb7 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -46,6 +46,7 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -53,6 +54,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.UUID; @@ -145,7 +147,7 @@ public Node( this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); - this.configDir = workingDir.resolve("config"); + this.configDir = Optional.ofNullable(spec.getConfigDir()).orElse(workingDir.resolve("config")); this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory this.debugPort = DefaultLocalClusterHandle.NEXT_DEBUG_PORT.getAndIncrement(); } @@ -293,6 +295,10 @@ Path getWorkingDir() { return workingDir; } + Path getConfigDir() { + return configDir; + } + public void waitUntilReady() { try { Retry.retryUntilTrue(NODE_UP_TIMEOUT, Duration.ofMillis(500), () -> { @@ -425,7 +431,7 @@ private void writeConfiguration() { try (Stream configFiles = Files.walk(distributionDir.resolve("config"))) { for (Path file : configFiles.toList()) { Path relativePath = distributionDir.resolve("config").relativize(file); - Path dest = configDir.resolve(relativePath); + Path dest = configDir.resolve(relativePath.toFile().getPath()); if (Files.exists(dest) == false) { Files.createDirectories(dest.getParent()); Files.copy(file, dest); @@ -567,6 +573,37 @@ private void writeSecureSecretsFile() { } } + private void updateRolesFileAtomically() throws IOException { + final Path targetRolesFile = workingDir.resolve("config").resolve("roles.yml"); + final Path tempFile = Files.createTempFile(workingDir.resolve("config"), null, null); + + // collect all roles.yml files that should be combined into a single roles file + final List rolesFiles = new ArrayList<>(spec.getRolesFiles().size() + 1); + rolesFiles.add(Resource.fromFile(distributionDir.resolve("config").resolve("roles.yml"))); + rolesFiles.addAll(spec.getRolesFiles()); + + // append all roles files to the temp file + rolesFiles.forEach(rolesFile -> { + try ( + Writer writer = Files.newBufferedWriter(tempFile, StandardOpenOption.APPEND); + Reader reader = new BufferedReader(new InputStreamReader(rolesFile.asStream())) + ) { + reader.transferTo(writer); + } catch (IOException e) { + throw new UncheckedIOException("Failed to append roles file " + rolesFile + " to " + tempFile, e); + } + }); + + // move the temp file to the target roles file atomically + try { + Files.move(tempFile, targetRolesFile, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new UncheckedIOException("Failed to move tmp roles file [" + tempFile + "] to [" + targetRolesFile + "]", e); + } finally { + Files.deleteIfExists(tempFile); + } + } + private void configureSecurity() { if (spec.isSecurityEnabled()) { if (spec.getUsers().isEmpty() == false) { @@ -576,13 +613,11 @@ private void configureSecurity() { if (resource instanceof MutableResource && roleFileListeners.add(resource)) { ((MutableResource) resource).addUpdateListener(updated -> { LOGGER.info("Updating roles.yml for node '{}'", name); - Path rolesFile = workingDir.resolve("config").resolve("roles.yml"); try { - Files.delete(rolesFile); - Files.copy(distributionDir.resolve("config").resolve("roles.yml"), rolesFile); - writeRolesFile(); + updateRolesFileAtomically(); + LOGGER.info("Successfully updated roles.yml for node '{}'", name); } catch (IOException e) { - throw new UncheckedIOException(e); + throw new UncheckedIOException("Failed to update roles.yml file for node [" + name + "]", e); } }); } @@ -610,7 +645,7 @@ private void configureSecurity() { if (operators.isEmpty() == false) { // TODO: Support service accounts here final String operatorUsersFileName = "operator_users.yml"; - final Path destination = workingDir.resolve("config").resolve(operatorUsersFileName); + final Path destination = configDir.resolve(operatorUsersFileName); if (Files.exists(destination)) { throw new IllegalStateException( "Operator users file [" @@ -637,7 +672,7 @@ private void configureSecurity() { } private void writeRolesFile() { - Path destination = workingDir.resolve("config").resolve("roles.yml"); + Path destination = configDir.resolve("roles.yml"); spec.getRolesFiles().forEach(rolesFile -> { try ( Writer writer = Files.newBufferedWriter(destination, StandardOpenOption.APPEND); @@ -654,7 +689,7 @@ private void installPlugins() { if (spec.getPlugins().isEmpty() == false) { Pattern pattern = Pattern.compile("(.+)(?:-\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?\\.zip)"); - LOGGER.info("Installing plugins {} into node '{}", spec.getPlugins(), name); + LOGGER.info("Installing plugins {} into node '{}", spec.getPlugins().keySet(), name); List pluginPaths = Arrays.stream(System.getProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP).split(File.pathSeparator)) .map(Path::of) .toList(); @@ -732,7 +767,7 @@ private void installPlugins() { private void installModules() { if (spec.getModules().isEmpty() == false) { - LOGGER.info("Installing modules {} into node '{}", spec.getModules(), name); + LOGGER.info("Installing modules {} into node '{}", spec.getModules().keySet(), name); List modulePaths = Arrays.stream(System.getProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP).split(File.pathSeparator)) .map(Path::of) .toList(); @@ -765,7 +800,12 @@ private void installModule(String moduleName, DefaultPluginInstallSpec installSp }); - IOUtils.syncMaybeWithLinks(modulePath, destination); + // If we aren't overriding anything we can use links here, otherwise do a full copy + if (installSpec.entitlementsOverride == null && installSpec.propertiesOverride == null) { + IOUtils.syncMaybeWithLinks(modulePath, destination); + } else { + IOUtils.syncWithCopy(modulePath, destination); + } try { if (installSpec.entitlementsOverride != null) { @@ -794,7 +834,9 @@ private void installModule(String moduleName, DefaultPluginInstallSpec installSp if (extendedProperty != null) { String[] extendedModules = extendedProperty.split(","); for (String module : extendedModules) { - installModule(module, new DefaultPluginInstallSpec(), modulePaths); + if (spec.getModules().containsKey(module) == false) { + installModule(module, new DefaultPluginInstallSpec(), modulePaths); + } } } } catch (IOException e) { @@ -820,7 +862,7 @@ private void startElasticsearch() { private Map getEnvironmentVariables() { Map environment = new HashMap<>(spec.resolveEnvironment()); - environment.put("ES_PATH_CONF", workingDir.resolve("config").toString()); + environment.put("ES_PATH_CONF", configDir.toString()); environment.put("ES_TMPDIR", workingDir.resolve("tmp").toString()); // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR environment.put("TMP", workingDir.resolve("tmp").toString()); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java index 7f1a384ebb43d..73c5afdec5b9f 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java @@ -212,7 +212,8 @@ private LocalNodeSpec build(LocalClusterSpec cluster, int nodeIndex) { getExtraConfigFiles(), getSystemPropertyProviders(), getSystemProperties(), - getJvmArgs() + getJvmArgs(), + Optional.ofNullable(getConfigDirSupplier()).map(Supplier::get).orElse(null) ); } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java index 9617cb633aa9b..82c5b1a54c13c 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; +import java.nio.file.Path; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -47,6 +48,7 @@ public abstract class AbstractLocalSpecBuilder> im private DistributionType distributionType; private Version version; private String keystorePassword; + private Supplier configDirSupplier; protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder parent) { this.parent = parent; @@ -270,12 +272,28 @@ public String getKeystorePassword() { return inherit(() -> parent.getKeystorePassword(), keystorePassword); } + @Override + public T withConfigDir(Supplier configDirSupplier) { + this.configDirSupplier = configDirSupplier; + return cast(this); + } + + public Supplier getConfigDirSupplier() { + return inherit(() -> parent.getConfigDirSupplier(), configDirSupplier); + } + @Override public T version(Version version) { this.version = version; return cast(this); } + @Override + public T version(String version) { + this.version = Version.fromString(version); + return cast(this); + } + public Version getVersion() { return inherit(() -> parent.getVersion(), version); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java index 4331728aedb1e..f2e2245e7321e 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java @@ -237,15 +237,15 @@ private WaitForHttpResource configureWaitForReady() throws MalformedURLException private void configureWaitSecurity(WaitForHttpResource wait, Node node) { String caFile = node.getSpec().getSetting("xpack.security.http.ssl.certificate_authorities", null); if (caFile != null) { - wait.setCertificateAuthorities(node.getWorkingDir().resolve("config").resolve(caFile).toFile()); + wait.setCertificateAuthorities(node.getConfigDir().resolve(caFile).toFile()); } String sslCertFile = node.getSpec().getSetting("xpack.security.http.ssl.certificate", null); if (sslCertFile != null) { - wait.setCertificateAuthorities(node.getWorkingDir().resolve("config").resolve(sslCertFile).toFile()); + wait.setCertificateAuthorities(node.getConfigDir().resolve(sslCertFile).toFile()); } String sslKeystoreFile = node.getSpec().getSetting("xpack.security.http.ssl.keystore.path", null); if (sslKeystoreFile != null && caFile == null) { // Can not set both trust stores and CA - wait.setTrustStoreFile(node.getWorkingDir().resolve("config").resolve(sslKeystoreFile).toFile()); + wait.setTrustStoreFile(node.getConfigDir().resolve(sslKeystoreFile).toFile()); } String keystorePassword = node.getSpec().getSetting("xpack.security.http.ssl.keystore.secure_password", null); if (keystorePassword != null) { @@ -254,7 +254,7 @@ private void configureWaitSecurity(WaitForHttpResource wait, Node node) { } private boolean isSecurityAutoConfigured(Node node) { - Path configFile = node.getWorkingDir().resolve("config").resolve("elasticsearch.yml"); + Path configFile = node.getConfigDir().resolve("elasticsearch.yml"); try (Stream lines = Files.lines(configFile)) { return lines.anyMatch(l -> l.contains("BEGIN SECURITY AUTO CONFIGURATION")); } catch (IOException e) { @@ -273,7 +273,7 @@ private void writeUnicastHostsFile() { LOGGER.info("Skipping writing unicast hosts file for node {}", node.getName()); return; } - Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt"); + Path hostsFile = node.getConfigDir().resolve("unicast_hosts.txt"); LOGGER.info("Writing unicast hosts file {} for node {}", hostsFile, node.getName()); Files.writeString(hostsFile, transportUris); } catch (IOException e) { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSystemPropertyProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSystemPropertyProvider.java index 62bbd10bcf851..fa1f386007192 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSystemPropertyProvider.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSystemPropertyProvider.java @@ -11,13 +11,21 @@ import org.elasticsearch.test.cluster.SystemPropertyProvider; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; -import static java.util.Map.entry; - public class DefaultSystemPropertyProvider implements SystemPropertyProvider { @Override public Map get(LocalClusterSpec.LocalNodeSpec nodeSpec) { - return Map.ofEntries(entry("ingest.geoip.downloader.enabled.default", "false"), entry("tests.testfeatures.enabled", "true")); + Map properties = new HashMap<>(); + properties.put("ingest.geoip.downloader.enabled.default", "false"); + + // enable test features unless we are running forwards compatibility tests + if (Boolean.parseBoolean(System.getProperty("tests.fwc", "false")) == false) { + properties.put("tests.testfeatures.enabled", "true"); + } + + return Collections.unmodifiableMap(properties); } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index ed5c0c5d1bbc0..cccf2a95234c9 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -19,6 +19,7 @@ import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -103,6 +104,7 @@ public static class LocalNodeSpec { private final List systemPropertyProviders; private final Map systemProperties; private final List jvmArgs; + private final Path configDir; private Version version; public LocalNodeSpec( @@ -124,7 +126,8 @@ public LocalNodeSpec( Map extraConfigFiles, List systemPropertyProviders, Map systemProperties, - List jvmArgs + List jvmArgs, + Path configDir ) { this.cluster = cluster; this.name = name; @@ -145,6 +148,7 @@ public LocalNodeSpec( this.systemPropertyProviders = systemPropertyProviders; this.systemProperties = systemProperties; this.jvmArgs = jvmArgs; + this.configDir = configDir; } void setVersion(Version version) { @@ -203,6 +207,10 @@ public List getJvmArgs() { return jvmArgs; } + public Path getConfigDir() { + return configDir; + } + public boolean isSecurityEnabled() { return Boolean.parseBoolean(getSetting("xpack.security.enabled", getVersion().onOrAfter("8.0.0") ? "true" : "false")); } @@ -339,7 +347,8 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider, Setting n.extraConfigFiles, n.systemPropertyProviders, n.systemProperties, - n.jvmArgs + n.jvmArgs, + n.configDir ) ) .toList(); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java index 20c92dc2b11d7..53d283224b26f 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java @@ -18,6 +18,7 @@ import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; +import java.nio.file.Path; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; @@ -130,6 +131,11 @@ interface LocalSpecBuilder> { */ T version(Version version); + /** + * Sets the version of Elasticsearch. Defaults to {@link Version#CURRENT}. + */ + T version(String version); + /** * Adds a system property to node JVM arguments. */ @@ -155,4 +161,10 @@ interface LocalSpecBuilder> { * Adds an additional command line argument to node JVM arguments. */ T jvmArg(String arg); + + /** + * Register a supplier to provide the config directory. The default config directory + * is used when the supplier is null or the return value of the supplier is null. + */ + T withConfigDir(Supplier configDirSupplier); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java index 048a3c49fcade..a672608aec440 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java @@ -37,8 +37,8 @@ public DistributionDescriptor resolve(Version version, DistributionType type) { ); } - // Snapshot distributions are never release builds and always use the default distribution - return new DefaultDistributionDescriptor(version, true, distributionDir, DistributionType.DEFAULT); + boolean isSnapshot = System.getProperty("tests.bwc.snapshot", "true").equals("false") == false; + return new DefaultDistributionDescriptor(version, isSnapshot, distributionDir, DistributionType.DEFAULT); } return delegate.resolve(version, type); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java index b1e2175205594..9289e47478e73 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java @@ -15,9 +15,12 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Comparator; import java.util.function.BiConsumer; import java.util.stream.Stream; @@ -118,35 +121,37 @@ public static void syncWithCopy(Path sourceRoot, Path destinationRoot) { private static void sync(Path sourceRoot, Path destinationRoot, BiConsumer syncMethod) { assert Files.exists(destinationRoot) == false; - try (Stream stream = Files.walk(sourceRoot)) { - stream.forEach(source -> { - Path relativeDestination = sourceRoot.relativize(source); - - Path destination = destinationRoot.resolve(relativeDestination); - if (Files.isDirectory(source)) { - try { - Files.createDirectories(destination); - } catch (IOException e) { - throw new UncheckedIOException("Can't create directory " + destination.getParent(), e); - } - } else { - try { - Files.createDirectories(destination.getParent()); - } catch (IOException e) { - throw new UncheckedIOException("Can't create directory " + destination.getParent(), e); - } + try { + Files.walkFileTree(sourceRoot, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path relativeDestination = sourceRoot.relativize(dir); + Path destination = destinationRoot.resolve(relativeDestination); + Files.createDirectories(destination); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path source, BasicFileAttributes attrs) throws IOException { + Path relativeDestination = sourceRoot.relativize(source); + Path destination = destinationRoot.resolve(relativeDestination); + Files.createDirectories(destination.getParent()); syncMethod.accept(destination, source); + return FileVisitResult.CONTINUE; } - }); - } catch (UncheckedIOException e) { - if (e.getCause() instanceof NoSuchFileException cause) { - // Ignore these files that are sometimes left behind by the JVM - if (cause.getFile() == null || cause.getFile().contains(".attach_pid") == false) { - throw new UncheckedIOException(cause); + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + if (exc instanceof NoSuchFileException noFileException) { + // Ignore these files that are sometimes left behind by the JVM + if (noFileException.getFile() != null && noFileException.getFile().contains(".attach_pid")) { + LOGGER.info("Ignoring file left behind by JVM: {}", noFileException.getFile()); + return FileVisitResult.CONTINUE; + } + } + throw exc; } - } else { - throw e; - } + }); } catch (IOException e) { throw new UncheckedIOException("Can't walk source " + sourceRoot, e); } diff --git a/test/test-clusters/src/test/java/org/elasticsearch/test/cluster/util/IOUtilsTests.java b/test/test-clusters/src/test/java/org/elasticsearch/test/cluster/util/IOUtilsTests.java new file mode 100644 index 0000000000000..dee1924d69a30 --- /dev/null +++ b/test/test-clusters/src/test/java/org/elasticsearch/test/cluster/util/IOUtilsTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.test.cluster.util; + +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.Is.isA; +import static org.junit.Assert.assertThrows; + +public class IOUtilsTests { + + @Test + public void testSyncWithLinks() throws IOException { + // given + Path sourceDir = Files.createTempDirectory("sourceDir"); + Files.createFile(sourceDir.resolve("file1.txt")); + Files.createFile(sourceDir.resolve("file2.txt")); + Files.createDirectory(sourceDir.resolve("nestedDir")); + Files.createFile(sourceDir.resolve("nestedDir").resolve("file3.txt")); + + Path baseDestinationDir = Files.createTempDirectory("baseDestinationDir"); + Path destinationDir = baseDestinationDir.resolve("destinationDir"); + + // when + IOUtils.syncWithLinks(sourceDir, destinationDir); + + // then + assertFileExists(destinationDir.resolve("file1.txt")); + assertFileExists(destinationDir.resolve("file2.txt")); + assertFileExists(destinationDir.resolve("nestedDir").resolve("file3.txt")); + } + + private void assertFileExists(Path path) throws IOException { + assertThat("File " + path + " doesn't exist", Files.exists(path), is(true)); + assertThat("File " + path + " is not a regular file", Files.isRegularFile(path), is(true)); + assertThat("File " + path + " is not readable", Files.isReadable(path), is(true)); + if (OS.current() != OS.WINDOWS) { + assertThat("Expected 2 hard links", Files.getAttribute(path, "unix:nlink"), is(2)); + } + } + + @Test + public void testSyncWithLinksThrowExceptionWhenDestinationIsNotWritable() throws IOException { + Assume.assumeFalse("On Windows read-only directories are not supported", OS.current() == OS.WINDOWS); + + // given + Path sourceDir = Files.createTempDirectory("sourceDir"); + Files.createFile(sourceDir.resolve("file1.txt")); + + Path baseDestinationDir = Files.createTempDirectory("baseDestinationDir"); + Path destinationDir = baseDestinationDir.resolve("destinationDir"); + + baseDestinationDir.toFile().setWritable(false); + + // when + UncheckedIOException ex = assertThrows(UncheckedIOException.class, () -> IOUtils.syncWithLinks(sourceDir, destinationDir)); + + // then + assertThat(ex.getCause(), isA(IOException.class)); + assertThat(ex.getCause().getMessage(), containsString("destinationDir")); + } +} diff --git a/test/yaml-rest-runner/build.gradle b/test/yaml-rest-runner/build.gradle index 37d2a00a68dda..1ae1315ac9ef7 100644 --- a/test/yaml-rest-runner/build.gradle +++ b/test/yaml-rest-runner/build.gradle @@ -1,6 +1,6 @@ apply plugin: 'elasticsearch.build' -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' dependencies { api project(':test:framework') diff --git a/test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java b/test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java index abd70d5c3a326..708f3d8e5293c 100644 --- a/test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java +++ b/test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java @@ -13,12 +13,17 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.junit.ClassRule; import java.io.IOException; public class ESClientYamlSuiteTestCaseFailLogIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + public ESClientYamlSuiteTestCaseFailLogIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -65,4 +70,9 @@ public void test() throws IOException { mockLog.assertAllExpectationsMatched(); } } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/build.gradle b/x-pack/build.gradle index da21ffc829d03..b7f38b61a61a0 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -26,7 +26,7 @@ subprojects { ext.xpackModule = { String moduleName -> ":x-pack:plugin:${moduleName}" } plugins.withType(PluginBuildPlugin).whenPluginAdded { - project.esplugin.licenseFile = rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt') + project.esplugin.licenseFile = layout.settingsDirectory.file('licenses/ELASTIC-LICENSE-2.0.txt').asFile project.esplugin.noticeFile = xpackRootProject.file('NOTICE.txt') } @@ -40,7 +40,7 @@ subprojects { } project.pluginManager.withPlugin("elasticsearch.build") { - project.ext.licenseFile.set(rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt')) + project.ext.licenseFile.set(layout.settingsDirectory.file('licenses/ELASTIC-LICENSE-2.0.txt').asFile) project.ext.noticeFile.set(xpackRootProject.file('NOTICE.txt')) } } diff --git a/x-pack/plugin/analytics/build.gradle b/x-pack/plugin/analytics/build.gradle index fff4d37349a17..c15687420cd3a 100644 --- a/x-pack/plugin/analytics/build.gradle +++ b/x-pack/plugin/analytics/build.gradle @@ -2,6 +2,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name = 'x-pack-analytics' @@ -14,6 +15,9 @@ base { } dependencies { + clusterModules project(':modules:aggregations') + clusterPlugins project(':x-pack:plugin:analytics') + api 'org.apache.commons:commons-math3:3.6.1' compileOnly project(path: xpackModule('core')) compileOnly project(":server") @@ -21,7 +25,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/analytics/src/javaRestTest/java/org/elasticsearch/multiterms/AggsTimeoutIT.java b/x-pack/plugin/analytics/src/javaRestTest/java/org/elasticsearch/multiterms/AggsTimeoutIT.java new file mode 100644 index 0000000000000..57bfc5f52ba09 --- /dev/null +++ b/x-pack/plugin/analytics/src/javaRestTest/java/org/elasticsearch/multiterms/AggsTimeoutIT.java @@ -0,0 +1,318 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.multiterms; + +import org.apache.http.client.config.RequestConfig; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.test.ListMatcher; +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; + +/** + * Runs slow aggregations with a timeout and asserts that they timeout and + * cancel the queries. + */ +public class AggsTimeoutIT extends ESRestTestCase { + private static final int DEPTH = 10; + private static final int VALUE_COUNT = 4; + private static final int TOTAL_DOCS = Math.toIntExact((long) Math.pow(VALUE_COUNT, DEPTH)); + private static final TimeValue TIMEOUT = TimeValue.timeValueSeconds(1); + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.INTEG_TEST) + .plugin("x-pack-analytics") + .module("aggregations") + .jvmArg("-Xmx1g") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testTerms() throws Exception { + Request request = new Request("POST", "/deep/_search"); + XContentBuilder body = JsonXContent.contentBuilder().prettyPrint().startObject(); + body.field("size", 0); + agg(body, "terms", 10); + request.setJsonEntity(Strings.toString(body.endObject())); + setTimeout(request); + try { + Map response = responseAsMap(client().performRequest(request)); + assertMap("not expected to finish", response, matchesMap()); + } catch (SocketTimeoutException timeout) { + logger.info("timed out"); + assertNoSearchesRunning(); + } + } + + private void agg(XContentBuilder body, String type, int depth) throws IOException { + if (depth == 0) { + return; + } + body.startObject("aggs").startObject(field("agg", depth)); + { + body.startObject(type); + body.field("field", field("kwd", depth - 1)); + body.endObject(); + } + agg(body, type, depth - 1); + body.endObject().endObject(); + } + + public void testMultiTerms() throws Exception { + Request request = new Request("POST", "/deep/_search"); + XContentBuilder body = JsonXContent.contentBuilder().prettyPrint().startObject(); + body.field("size", 0); + autoDateInMultiTerms(body, b -> { + for (int i = 0; i < DEPTH; i++) { + b.startObject().field("field", field("kwd", i)).endObject(); + } + }); + request.setJsonEntity(Strings.toString(body.endObject())); + setTimeout(request); + try { + Map response = responseAsMap(client().performRequest(request)); + ListMatcher buckets = matchesList(); + for (int i = 0; i < 10; i++) { + buckets = buckets.item( + matchesMap().entry("key_as_string", any(String.class)) + .entry("key", hasSize(10)) + .entry("doc_count", 1) + .entry("adh", matchesMap().entry("buckets", hasSize(1)).entry("interval", "1s")) + ); + } + MapMatcher agg = matchesMap().entry("buckets", buckets) + .entry("doc_count_error_upper_bound", 0) + .entry("sum_other_doc_count", greaterThan(0)); + assertMap(response, matchesMap().extraOk().entry("aggregations", matchesMap().entry("multi", agg))); + } catch (SocketTimeoutException timeout) { + logger.info("timed out"); + assertNoSearchesRunning(); + } + } + + public void testMultiTermWithTimestamp() throws Exception { + Request request = new Request("POST", "/deep/_search"); + XContentBuilder body = JsonXContent.contentBuilder().prettyPrint().startObject(); + body.field("size", 0); + autoDateInMultiTerms(body, b -> { + b.startObject().field("field", field("kwd", 0)).endObject(); + b.startObject().field("field", "@timestamp").endObject(); + }); + request.setJsonEntity(Strings.toString(body.endObject())); + setTimeout(request); + try { + Map response = responseAsMap(client().performRequest(request)); + ListMatcher buckets = matchesList(); + for (int i = 0; i < 10; i++) { + buckets = buckets.item( + matchesMap().entry("key_as_string", any(String.class)) + .entry("key", hasSize(10)) + .entry("doc_count", 1) + .entry("adh", matchesMap().entry("buckets", hasSize(1)).entry("interval", "1s")) + ); + } + MapMatcher agg = matchesMap().entry("buckets", buckets) + .entry("doc_count_error_upper_bound", 0) + .entry("sum_other_doc_count", greaterThan(0)); + assertMap(response, matchesMap().extraOk().entry("aggregations", matchesMap().entry("multi", agg))); + } catch (SocketTimeoutException timeout) { + logger.info("timed out"); + assertNoSearchesRunning(); + } + } + + private void autoDateInMultiTerms(XContentBuilder body, CheckedConsumer terms) throws IOException { + body.startObject("aggs").startObject("multi"); + { + body.startObject("multi_terms"); + { + body.startArray("terms"); + terms.accept(body); + body.endArray(); + body.startArray("order"); + { + body.startObject().field("_count", "desc").endObject(); + body.startObject().field("_key", "asc").endObject(); + } + body.endArray(); + } + body.endObject(); + body.startObject("aggs").startObject("adh").startObject("auto_date_histogram"); + { + body.field("field", "@timestamp"); + body.field("buckets", 1); + } + body.endObject().endObject().endObject(); + } + body.endObject().endObject(); + } + + @Before + public void createDeep() throws IOException { + if (indexExists("deep")) { + return; + } + logger.info("creating deep index"); + XContentBuilder mapping = JsonXContent.contentBuilder().startObject(); + mapping.startObject("properties"); + mapping.startObject("@timestamp").field("type", "date").endObject(); + for (int f = 0; f < DEPTH; f++) { + mapping.startObject(field("kwd", f)).field("type", "keyword").endObject(); + } + CreateIndexResponse createIndexResponse = createIndex( + "deep", + Settings.builder().put("index.number_of_replicas", 0).build(), + Strings.toString(mapping.endObject().endObject()) + ); + assertThat(createIndexResponse.isAcknowledged(), equalTo(true)); + Bulk bulk = new Bulk(); + bulk.doc(new StringBuilder("{"), 0); + bulk.flush(); + + MapMatcher shardsOk = matchesMap().entry("total", 1).entry("failed", 0).entry("successful", 1); + logger.info("refreshing deep index"); + Map refresh = responseAsMap(client().performRequest(new Request("POST", "/_refresh"))); + assertMap(refresh, matchesMap().entry("_shards", shardsOk)); + + logger.info("double checking deep index count"); + Map count = responseAsMap(client().performRequest(new Request("POST", "/deep/_count"))); + assertMap(count, matchesMap().entry("_shards", shardsOk.entry("skipped", 0)).entry("count", TOTAL_DOCS)); + + logger.info("deep index ready for test"); + } + + private String field(String prefix, int field) { + return String.format(Locale.ROOT, "%s%03d", prefix, field); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + class Bulk { + private static final int BULK_SIZE = Math.toIntExact(ByteSizeValue.ofMb(2).getBytes()); + + StringBuilder bulk = new StringBuilder(); + int current = 0; + int total = 0; + long timestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2025-01-01T00:00:00Z"); + + void doc(StringBuilder doc, int field) throws IOException { + if (field != 0) { + doc.append(','); + } + int len = doc.length(); + for (int value = 0; value < VALUE_COUNT; value++) { + doc.append('"').append(field("kwd", field)).append("\":\"").append(value).append('"'); + if (field == DEPTH - 1) { + doc.append(",\"@timestamp\":").append(timestamp).append('}'); + timestamp += TimeValue.timeValueMinutes(1).millis(); + addToBulk(doc); + } else { + doc(doc, field + 1); + } + doc.setLength(len); + } + } + + void addToBulk(StringBuilder doc) throws IOException { + current++; + total++; + bulk.append("{\"index\":{}}\n"); + bulk.append(doc).append('\n'); + if (bulk.length() > BULK_SIZE) { + flush(); + } + } + + void flush() throws IOException { + logger.info( + "Flushing to deep {} docs/{}. Total {}% {}/{}", + current, + ByteSizeValue.ofBytes(bulk.length()), + String.format(Locale.ROOT, "%04.1f", 100.0 * total / TOTAL_DOCS), + total, + TOTAL_DOCS + ); + Request request = new Request("POST", "/deep/_bulk"); + request.setJsonEntity(bulk.toString()); + Map response = responseAsMap(client().performRequest(request)); + assertMap(response, matchesMap().extraOk().entry("errors", false)); + bulk.setLength(0); + current = 0; + } + } + + private void setTimeout(Request request) { + RequestConfig.Builder config = RequestConfig.custom(); + config.setSocketTimeout(Math.toIntExact(TIMEOUT.millis())); + request.setOptions(request.getOptions().toBuilder().setRequestConfig(config.build())); + } + + /** + * Asserts that within a minute the _search has left the _tasks api. + *

+ * It'd sure be more convenient if, whenever the _search has returned + * back to us the _tasks API doesn't contain the _search. But sometimes + * it still does. So long as it stops eventually that's + * still indicative of the interrupt code working. + *

+ */ + private void assertNoSearchesRunning() throws Exception { + assertBusy(() -> { + Request tasks = new Request("GET", "/_tasks"); + tasks.addParameter("actions", "*search"); + tasks.addParameter("detailed", ""); + assertBusy(() -> { + Map response = responseAsMap(client().performRequest(tasks)); + // If there are running searches the map in `nodes` is non-empty. + if (response.isEmpty() == false) { + logger.warn("search still running, hot threads:\n{}", hotThreads()); + } + assertMap(response, matchesMap().entry("nodes", matchesMap())); + }); + }, 1, TimeUnit.MINUTES); + } + + private String hotThreads() throws IOException { + Request tasks = new Request("GET", "/_nodes/hot_threads"); + return EntityUtils.toString(client().performRequest(tasks).getEntity()); + } +} diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java index 39a6fa1e4b34f..7e12ee1353d29 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchErrorTraceIT.java @@ -7,17 +7,23 @@ package org.elasticsearch.xpack.search; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.ErrorTraceHelper; +import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.transport.TransportMessageListener; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Collection; @@ -40,6 +46,11 @@ protected Collection> nodePlugins() { private AtomicBoolean transportMessageHasStackTrace; + @BeforeClass + public static void setDebugLogLevel() { + Configurator.setLevel(SearchService.class, Level.DEBUG); + } + @Before private void setupMessageListener() { internalCluster().getDataNodeInstances(TransportService.class).forEach(ts -> { @@ -154,6 +165,85 @@ public void testAsyncSearchFailingQueryErrorTraceFalse() throws IOException, Int assertFalse(transportMessageHasStackTrace.get()); } + public void testDataNodeDoesNotLogStackTraceWhenErrorTraceTrue() throws IOException, InterruptedException { + transportMessageHasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + Request searchRequest = new Request("POST", "/_async_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + searchRequest.addParameter("error_trace", "true"); + searchRequest.addParameter("keep_on_completion", "true"); + searchRequest.addParameter("wait_for_completion_timeout", "0ms"); + + String errorTriggeringIndex = "test2"; + int numShards = getNumShards(errorTriggeringIndex).numPrimaries; + try (var mockLog = MockLog.capture(SearchService.class)) { + ErrorTraceHelper.addUnseenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); + + Map responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO); + String asyncExecutionId = (String) responseEntity.get("id"); + Request request = new Request("GET", "/_async_search/" + asyncExecutionId); + request.addParameter("error_trace", "true"); + while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) { + responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L)); + } + + mockLog.assertAllExpectationsMatched(); + } + } + + public void testDataNodeLogsStackTraceWhenErrorTraceFalseOrEmpty() throws IOException, InterruptedException { + transportMessageHasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + // error_trace defaults to false so we can test both cases with some randomization + final boolean defineErrorTraceFalse = randomBoolean(); + + Request searchRequest = new Request("POST", "/_async_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + if (defineErrorTraceFalse) { + searchRequest.addParameter("error_trace", "false"); + } + searchRequest.addParameter("keep_on_completion", "true"); + searchRequest.addParameter("wait_for_completion_timeout", "0ms"); + + String errorTriggeringIndex = "test2"; + int numShards = getNumShards(errorTriggeringIndex).numPrimaries; + try (var mockLog = MockLog.capture(SearchService.class)) { + ErrorTraceHelper.addSeenLoggingExpectations(numShards, mockLog, errorTriggeringIndex); + + Map responseEntity = performRequestAndGetResponseEntityAfterDelay(searchRequest, TimeValue.ZERO); + String asyncExecutionId = (String) responseEntity.get("id"); + Request request = new Request("GET", "/_async_search/" + asyncExecutionId); + if (defineErrorTraceFalse) { + request.addParameter("error_trace", "false"); + } + while (responseEntity.get("is_running") instanceof Boolean isRunning && isRunning) { + responseEntity = performRequestAndGetResponseEntityAfterDelay(request, TimeValue.timeValueSeconds(1L)); + } + + mockLog.assertAllExpectationsMatched(); + } + } + public void testAsyncSearchFailingQueryErrorTraceFalseOnSubmitAndTrueOnGet() throws IOException, InterruptedException { transportMessageHasStackTrace = new AtomicBoolean(); setupIndexWithDocs(); diff --git a/x-pack/plugin/blob-cache/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/blob-cache/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..c24a99d12136f --- /dev/null +++ b/x-pack/plugin/blob-cache/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,5 @@ +org.elasticsearch.blobcache: + - files: + - relative_path: "" + relative_to: data + mode: read_write diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 598cc3b5bef2f..55f2e322528e4 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -38,7 +38,7 @@ artifacts { def restTestBlacklist = [] // TODO: fix this rest test to not depend on a hardcoded port! restTestBlacklist.addAll(['getting_started/10_monitor_cluster_health/*']) -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { // these tests attempt to install basic/internal licenses signed against the dev/public.key // Since there is no infrastructure in place (anytime soon) to generate licenses using the production // private key, these tests are blacklisted in non-snapshot test runs @@ -224,3 +224,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> task.skipTest("esql/40_unsupported_types/unsupported", "TODO: support for subset of metric fields") task.skipTest("esql/40_unsupported_types/unsupported with sort", "TODO: support for subset of metric fields") }) + +tasks.named('yamlRestTestV7CompatTest').configure { + systemProperty 'es.queryable_built_in_roles_enabled', 'false' +} diff --git a/x-pack/plugin/ccr/build.gradle b/x-pack/plugin/ccr/build.gradle index f4ade5b441877..539fb24fdb07d 100644 --- a/x-pack/plugin/ccr/build.gradle +++ b/x-pack/plugin/ccr/build.gradle @@ -1,46 +1,68 @@ +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-test-artifact' + esplugin { name = 'x-pack-ccr' description = 'Elasticsearch Expanded Pack Plugin - CCR' - classname ='org.elasticsearch.xpack.ccr.Ccr' - hasNativeController =false - requiresKeystore =true + classname = 'org.elasticsearch.xpack.ccr.Ccr' + hasNativeController = false + requiresKeystore = true extendedPlugins = ['x-pack-core'] } + base { archivesName = 'x-pack-ccr' } -// Integration Test classes that cannot run with the security manager -String[] noSecurityManagerITClasses = ["**/CloseFollowerIndexIT.class"] +dependencies { + compileOnly project(":server") -tasks.register('internalClusterTestNoSecurityManager', Test) { - testClassesDirs = sourceSets.internalClusterTest.output.classesDirs - classpath = sourceSets.internalClusterTest.runtimeClasspath - include noSecurityManagerITClasses - systemProperty 'tests.security.manager', 'false' + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation(testArtifact(project(xpackModule('monitoring')))) + testImplementation(project(":modules:analysis-common")) + testImplementation(project(":modules:data-streams")) + javaRestTestImplementation(testImplementation(testArtifact(project(xpackModule('core'))))) + + clusterModules project(":modules:analysis-common") + clusterModules project(":modules:mapper-extras") + clusterModules project(":modules:data-streams") + clusterModules project(":modules:ingest-common") + clusterModules project(xpackModule("monitoring")) + clusterModules project(xpackModule("ilm")) + clusterModules project(xpackModule("wildcard")) + clusterModules project(xpackModule("stack")) + clusterModules project(xpackModule("mapper-constant-keyword")) + clusterModules project(xpackModule("searchable-snapshots")) } -tasks.named("check").configure { dependsOn 'internalClusterTestNoSecurityManager' } -tasks.named('internalClusterTest').configure { - exclude noSecurityManagerITClasses +restResources { + restApi { + include '_common', 'cluster', 'nodes', 'indices', 'index', 'info', 'ccr' + } } -tasks.named('internalClusterTestTestingConventions').configure { +tasks.named('internalClusterTest') { + systemProperty 'tests.security.manager', 'false' +} + +tasks.named('internalClusterTestTestingConventions') { baseClass 'org.elasticsearch.xpack.CcrIntegTestCase' baseClass 'org.elasticsearch.xpack.CcrSingleNodeTestCase' baseClass 'org.elasticsearch.test.ESIntegTestCase' } -addQaCheckDependencies(project) - -dependencies { - compileOnly project(":server") +tasks.named("yamlRestTest") { + usesDefaultDistribution("uses _xpack info api") +} - compileOnly project(path: xpackModule('core')) - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation(testArtifact(project(xpackModule('monitoring')))) - testImplementation(project(":modules:analysis-common")) - testImplementation(project(":modules:data-streams")) +tasks.withType(StandaloneRestIntegTestTask).configureEach { + // These fail in CI but only when run as part of checkPart2 and not individually. + // Tracked in : https://github.com/elastic/elasticsearch/issues/66661 + buildParams.withFipsEnabledOnly(it) } diff --git a/x-pack/plugin/ccr/qa/build.gradle b/x-pack/plugin/ccr/qa/build.gradle index 4be504e616920..d5bc38d2e8dd5 100644 --- a/x-pack/plugin/ccr/qa/build.gradle +++ b/x-pack/plugin/ccr/qa/build.gradle @@ -1,4 +1,9 @@ -import org.elasticsearch.gradle.internal.info.BuildParams +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ apply plugin: 'elasticsearch.java' diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle deleted file mode 100644 index c1c683e95a2ec..0000000000000 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle +++ /dev/null @@ -1,78 +0,0 @@ -import org.elasticsearch.gradle.internal.test.RestIntegTestTask -import org.elasticsearch.gradle.testclusters.TestClusterValueSource -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.testclusters.TestClustersRegistry -import org.elasticsearch.gradle.util.GradleUtils - -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' - -dependencies { - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation project(xpackModule('ccr')) - testImplementation project(':x-pack:plugin:ccr:qa') -} - -def clusterPath = getPath() - -def leaderCluster = testClusters.register("leader-cluster") { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' -} - -def followCluster = testClusters.register("follow-cluster") { - testDistribution = 'DEFAULT' - setting 'xpack.monitoring.collection.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - Provider serviceProvider = GradleUtils.getBuildService( - project.gradle.sharedServices, - TestClustersPlugin.REGISTRY_SERVICE_NAME - ) - def leaderInfo = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("leader-cluster") - it.parameters.service = serviceProvider - } - def leaderUris = leaderInfo.map { it.getAllTransportPortURI() } - - setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE -} - -tasks.register("leader-cluster", RestIntegTestTask) { - mustRunAfter("precommit") - systemProperty 'tests.target_cluster', 'leader' -} - -File policyFile = file("${buildDir}/tmp/java.policy") -tasks.register("writeJavaPolicy") { - doLast { - if (policyFile.parentFile.exists() == false && policyFile.parentFile.mkdirs() == false) { - throw new GradleException("failed to create temporary directory [${tmp}]") - } - policyFile.write( - [ - "grant {", - " permission java.io.FilePermission \"${-> followCluster.map { it.getFirstNode().getServerLog() }.get()}\", \"read\";", - "};" - ].join("\n") - ) - } -} - -tasks.register("follow-cluster", RestIntegTestTask) { - dependsOn 'writeJavaPolicy', "leader-cluster" - useCluster leaderCluster - systemProperty 'tests.target_cluster', 'follow' - nonInputProperties.systemProperty 'java.security.policy', "file://${policyFile}" - nonInputProperties.systemProperty 'tests.leader_host', getClusterInfo('leader-cluster').map { it.getAllHttpSocketURI().get(0) } - nonInputProperties.systemProperty 'log', followCluster.map(c -> c.getFirstNode().getServerLog()) -} - -tasks.named("check").configure { dependsOn "follow-cluster" } diff --git a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle index cdef49e037623..cbf8da1a4296c 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle @@ -1,111 +1,20 @@ -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.test.RestIntegTestTask -import org.elasticsearch.gradle.testclusters.TestClusterValueSource -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.testclusters.TestClustersRegistry -import org.elasticsearch.gradle.util.GradleUtils - -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +apply plugin: 'elasticsearch.internal-java-rest-test' dependencies { - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation project(xpackModule('ccr')) - testImplementation project(':x-pack:plugin:ccr:qa') -} - -def clusterPath = getPath() -def leaderCluster = testClusters.register('leader-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - setting 'path.repo', "${layout.buildDirectory.asFile.get()}/cluster/shared/repo/leader-cluster" -} - -def middleCluster = testClusters.register('middle-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - - Provider serviceProvider = GradleUtils.getBuildService( - project.gradle.sharedServices, - TestClustersPlugin.REGISTRY_SERVICE_NAME - ) - def leaderInfo = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("leader-cluster") - it.parameters.service = serviceProvider - } - def leaderUris = leaderInfo.map { it.getAllTransportPortURI() } - setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE -} - -tasks.register("leader-cluster", RestIntegTestTask) { - mustRunAfter("precommit") - systemProperty 'tests.target_cluster', 'leader' - systemProperty 'tests.leader_cluster_repository_path', "${layout.buildDirectory.asFile.get()}/cluster/shared/repo/leader-cluster" -} - -tasks.register("middle-cluster", RestIntegTestTask) { - dependsOn "leader-cluster" - useCluster testClusters.named("leader-cluster") - systemProperty 'tests.target_cluster', 'middle' - systemProperty 'tests.leader_cluster_repository_path', "${layout.buildDirectory.asFile.get()}/cluster/shared/repo/leader-cluster" - - def leaderUri = getClusterInfo('leader-cluster').map { it.allHttpSocketURI.get(0) } - nonInputProperties.systemProperty 'tests.leader_host', leaderUri + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(xpackModule('ccr')), 'javaRestTest')) + javaRestTestImplementation project(xpackModule('ccr')) } -tasks.register('follow-cluster', RestIntegTestTask) { - dependsOn "leader-cluster", "middle-cluster" - useCluster leaderCluster - useCluster middleCluster - systemProperty 'tests.target_cluster', 'follow' - systemProperty 'tests.leader_cluster_repository_path', "${layout.buildDirectory.asFile.get()}/cluster/shared/repo/leader-cluster" - - def leaderUri = getClusterInfo('leader-cluster').map { it.allHttpSocketURI.get(0) } - def middleUri = getClusterInfo('middle-cluster').map { it.allHttpSocketURI.get(0) } - nonInputProperties.systemProperty 'tests.leader_host', leaderUri - nonInputProperties.systemProperty 'tests.middle_host', middleUri +tasks.named("javaRestTest") { + usesDefaultDistribution("uses _xpack usage api") + // These fail in CI but only when run as part of checkPart2 and not individually. + // Tracked in : https://github.com/elastic/elasticsearch/issues/66661 + buildParams.withFipsEnabledOnly(it) } - -testClusters.matching { it.name == "follow-cluster" }.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.monitoring.collection.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - - Provider serviceProvider = GradleUtils.getBuildService( - project.gradle.sharedServices, - TestClustersPlugin.REGISTRY_SERVICE_NAME - ) - def leaderUris = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("leader-cluster") - it.parameters.service = serviceProvider - }.map { it.getAllTransportPortURI() } - - def middleUris = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("middle-cluster") - it.parameters.service = serviceProvider - }.map { it.getAllTransportPortURI() } - - setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE - setting 'cluster.remote.middle_cluster.seeds', - { "\"${middleUris.get().join(",")}\"" }, IGNORE_VALUE -} - - -testClusters.configureEach { - requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") -} - -tasks.named("check").configure { dependsOn "follow-cluster" } diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java similarity index 71% rename from x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java rename to x-pack/plugin/ccr/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java index 170f34be15e56..9aabaef93a793 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java @@ -6,12 +6,20 @@ */ package org.elasticsearch.xpack.ccr; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.xcontent.ObjectPath; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; import java.io.IOException; import java.util.Map; @@ -20,10 +28,45 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; -public class XPackUsageIT extends ESCCRRestTestCase { +public class XPackUsageIT extends AbstractCCRRestTestCase { + + public static LocalClusterConfigProvider commonConfig = c -> c.distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderCluster).around(followerCluster); + + public XPackUsageIT(@Name("targetCluster") TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } public void testXPackCcrUsage() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { logger.info("skipping test, waiting for target cluster [follow]"); return; } diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java deleted file mode 100644 index f2b431ed1c8b7..0000000000000 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ccr; - -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; - -public class ChainIT extends ESCCRRestTestCase { - - public void testFollowIndex() throws Exception { - final int numDocs = 128; - final String leaderIndexName = "leader"; - final String middleIndexName = "middle"; - if ("leader".equals(targetCluster)) { - logger.info("Running against leader cluster"); - String mapping = ""; - if (randomBoolean()) { // randomly do source filtering on indexing - mapping = """ - "_source": { "includes": ["field"], "excludes": ["filtered_field"]}"""; - } - createIndex(adminClient(), leaderIndexName, Settings.EMPTY, mapping, null); - for (int i = 0; i < numDocs; i++) { - logger.info("Indexing doc [{}]", i); - index(client(), leaderIndexName, Integer.toString(i), "field", i, "filtered_field", "true"); - } - refresh(adminClient(), leaderIndexName); - verifyDocuments(leaderIndexName, numDocs, "filtered_field:true"); - } else if ("middle".equals(targetCluster)) { - logger.info("Running against middle cluster"); - followIndex("leader_cluster", leaderIndexName, middleIndexName); - assertBusy(() -> verifyDocuments(middleIndexName, numDocs, "filtered_field:true")); - try (RestClient leaderClient = buildLeaderClient()) { - int id = numDocs; - index(leaderClient, leaderIndexName, Integer.toString(id), "field", id, "filtered_field", "true"); - index(leaderClient, leaderIndexName, Integer.toString(id + 1), "field", id + 1, "filtered_field", "true"); - index(leaderClient, leaderIndexName, Integer.toString(id + 2), "field", id + 2, "filtered_field", "true"); - } - assertBusy(() -> verifyDocuments(middleIndexName, numDocs + 3, "filtered_field:true")); - } else if ("follow".equals(targetCluster)) { - logger.info("Running against follow cluster"); - final String followIndexName = "follow"; - followIndex("middle_cluster", middleIndexName, followIndexName); - assertBusy(() -> verifyDocuments(followIndexName, numDocs + 3, "filtered_field:true")); - - try (RestClient leaderClient = buildLeaderClient()) { - int id = numDocs + 3; - index(leaderClient, leaderIndexName, Integer.toString(id), "field", id, "filtered_field", "true"); - index(leaderClient, leaderIndexName, Integer.toString(id + 1), "field", id + 1, "filtered_field", "true"); - index(leaderClient, leaderIndexName, Integer.toString(id + 2), "field", id + 2, "filtered_field", "true"); - } - - try (RestClient middleClient = buildMiddleClient()) { - assertBusy(() -> verifyDocuments(middleIndexName, numDocs + 6, "filtered_field:true", middleClient)); - } - - assertBusy(() -> verifyDocuments(followIndexName, numDocs + 6, "filtered_field:true")); - } else { - fail("unexpected target cluster [" + targetCluster + "]"); - } - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - -} diff --git a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle deleted file mode 100644 index ad4d2cb5afc7c..0000000000000 --- a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle +++ /dev/null @@ -1,60 +0,0 @@ -import org.elasticsearch.gradle.internal.test.RestIntegTestTask -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE -import org.elasticsearch.gradle.testclusters.TestClusterValueSource -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.testclusters.TestClustersRegistry -import org.elasticsearch.gradle.util.GradleUtils - -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' - -dependencies { - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation project(xpackModule('ccr')) - testImplementation project(':x-pack:plugin:ccr:qa:') -} - -def clusterPath = getPath() - -def leaderCluster = testClusters.register('leader-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' -} - -def followerCluster = testClusters.register('follow-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - - Provider serviceProvider = GradleUtils.getBuildService( - project.gradle.sharedServices, - TestClustersPlugin.REGISTRY_SERVICE_NAME - ) - def leaderInfo = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("leader-cluster") - it.parameters.service = serviceProvider - } - def leaderUris = leaderInfo.map { it.getAllTransportPortURI() } - - setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE -} - -tasks.register('leader-cluster', RestIntegTestTask) { - mustRunAfter("precommit") - systemProperty 'tests.target_cluster', 'leader' -} - -tasks.register('follow-cluster', RestIntegTestTask) { - dependsOn 'leader-cluster' - useCluster leaderCluster - systemProperty 'tests.target_cluster', 'follow' - - def followUri = getClusterInfo('follow-cluster').map { it.allHttpSocketURI.get(0) } - nonInputProperties.systemProperty 'tests.leader_host', followUri -} - -tasks.named("check").configure { dependsOn "follow-cluster" } diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle deleted file mode 100644 index 5f173b8831df8..0000000000000 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' - -restResources { - restApi { - include '_common', 'cluster', 'nodes', 'indices', 'index', 'info', 'ccr' - } -} - -dependencies { - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - // Disable assertions in FollowingEngineAssertions, otherwise an AssertionError is thrown before - // indexing a document directly in a follower index. In a rest test we like to test the exception - // that is thrown in production when indexing a document directly in a follower index. - jvmArgs '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.security.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - // TODO: reduce the need for superuser here - user username: 'ccr-user', password: 'ccr-user-password', role: 'superuser' -} diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle deleted file mode 100644 index 89ad8cad84987..0000000000000 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ /dev/null @@ -1,79 +0,0 @@ -import org.elasticsearch.gradle.internal.test.RestIntegTestTask -import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE -import org.elasticsearch.gradle.testclusters.TestClusterValueSource -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.testclusters.TestClustersRegistry -import org.elasticsearch.gradle.util.GradleUtils - -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' - -dependencies { - testImplementation project(':x-pack:plugin:ccr:qa') -} - -def clusterPath = getPath() - -def leaderCluster = testClusters.register('leader-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' -} - -def followCluster = testClusters.register('follow-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.monitoring.collection.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - - Provider serviceProvider = GradleUtils.getBuildService( - project.gradle.sharedServices, - TestClustersPlugin.REGISTRY_SERVICE_NAME - ) - def leaderInfo = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("leader-cluster") - it.parameters.service = serviceProvider - } - def leaderUri = leaderInfo.map { it.getAllTransportPortURI().get(0) } - - setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderUri.get()}\"" }, IGNORE_VALUE - nameCustomization = { 'follow' } -} - -tasks.register('leader-cluster', RestIntegTestTask) { - mustRunAfter("precommit") - systemProperty 'tests.target_cluster', 'leader' -} - -tasks.register('follow-cluster', RestIntegTestTask) { - dependsOn 'leader-cluster' - useCluster leaderCluster - systemProperty 'tests.target_cluster', 'follow' - - def leaderUri = getClusterInfo("leader-cluster").map { it.allHttpSocketURI.get(0) } - nonInputProperties.systemProperty 'tests.leader_host', leaderUri -} - -tasks.register("followClusterRestartTest", StandaloneRestIntegTestTask) { - dependsOn 'follow-cluster' - useCluster leaderCluster - useCluster followCluster - - systemProperty 'tests.rest.load_packaged', 'false' - systemProperty 'tests.target_cluster', 'follow-restart' - def leaderUri = getClusterInfo('leader-cluster').map { it.allHttpSocketURI.get(0) } - def followUris = getClusterInfo('follow-cluster').map { it.allHttpSocketURI.join(",") } - nonInputProperties.systemProperty 'tests.leader_host', leaderUri - nonInputProperties.systemProperty 'tests.rest.cluster', followUris - - doFirst { - getRegistry().get().restart(clusterPath, "follow-cluster") - } -} - -tasks.named("check").configure { dependsOn "followClusterRestartTest" } diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle deleted file mode 100644 index 3ceb86a632e0a..0000000000000 --- a/x-pack/plugin/ccr/qa/security/build.gradle +++ /dev/null @@ -1,65 +0,0 @@ -import org.elasticsearch.gradle.internal.test.RestIntegTestTask -import org.elasticsearch.gradle.testclusters.TestClusterValueSource -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.testclusters.TestClustersRegistry -import org.elasticsearch.gradle.util.GradleUtils - -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' - -dependencies { - testImplementation(testArtifact(project(xpackModule('core')))) - testImplementation project(path: xpackModule('ccr')) - testImplementation project(':x-pack:plugin:ccr:qa') -} - -def clusterPath = getPath() - -def leadCluster = testClusters.register('leader-cluster') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - extraConfigFile 'roles.yml', file('leader-roles.yml') - user username: "test_admin", role: "superuser" - user username: "test_ccr", role: "ccruser" -} - -testClusters.register('follow-cluster') { - testDistribution = 'DEFAULT' - Provider serviceProvider = GradleUtils.getBuildService( - project.gradle.sharedServices, - TestClustersPlugin.REGISTRY_SERVICE_NAME - ) - def leaderUris = project.getProviders().of(TestClusterValueSource.class) { - it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("leader-cluster") - it.parameters.service = serviceProvider - }.map { it.AllTransportPortURI } - - setting 'cluster.remote.leader_cluster.seeds', { - "\"${leaderUris.get().join(",")}\"" - }, IGNORE_VALUE - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - setting 'xpack.monitoring.collection.enabled', 'false' // will be enabled by tests - extraConfigFile 'roles.yml', file('follower-roles.yml') - user username: "test_admin", role: "superuser" - user username: "test_ccr", role: "ccruser" -} - -tasks.register('leader-cluster', RestIntegTestTask) { - mustRunAfter("precommit") - systemProperty 'tests.target_cluster', 'leader' -} - -def followerClusterTestTask = tasks.register('follow-cluster', RestIntegTestTask) { - dependsOn 'leader-cluster' - useCluster leadCluster - systemProperty 'tests.target_cluster', 'follow' - def leaderUri = getClusterInfo('leader-cluster').map { it.allHttpSocketURI.get(0) } - nonInputProperties.systemProperty 'tests.leader_host', leaderUri -} - -tasks.named("check").configure { dependsOn(followerClusterTestTask) } diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java new file mode 100644 index 0000000000000..81f84c6b0cf0a --- /dev/null +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java @@ -0,0 +1,467 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.ccr; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.apache.http.HttpHost; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.LazyInitializable; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; + +@TestCaseOrdering(AbstractCCRRestTestCase.TargetClusterTestOrdering.class) +public abstract class AbstractCCRRestTestCase extends ESRestTestCase { + + protected final TargetCluster targetCluster; + private static TargetCluster clientTargetCluster; + + public AbstractCCRRestTestCase(@Name("targetCluster") TargetCluster targetCluster) { + this.targetCluster = targetCluster; + } + + protected static List leaderMiddleFollower() { + return Arrays.stream(TargetCluster.values()).map(v -> new Object[] { v }).toList(); + } + + protected static List leaderFollower() { + return Arrays.stream(TargetCluster.values()).filter(c -> c != TargetCluster.MIDDLE).map(v -> new Object[] { v }).toList(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + protected abstract ElasticsearchCluster getLeaderCluster(); + + protected abstract ElasticsearchCluster getFollowerCluster(); + + protected ElasticsearchCluster getMiddleCluster() { + throw new UnsupportedOperationException("cannot get middle cluster"); + }; + + @Override + protected String getTestRestCluster() { + clientTargetCluster = targetCluster; + + return switch (targetCluster) { + case LEADER -> getLeaderCluster().getHttpAddresses(); + case MIDDLE -> getMiddleCluster().getHttpAddresses(); + case FOLLOWER -> getFollowerCluster().getHttpAddresses(); + }; + } + + @Before + public void maybeReInitClient() throws Exception { + if (clientTargetCluster != targetCluster) { + closeClients(); + initClient(); + } + } + + protected static void index(String index, String id, Object... fields) throws IOException { + index(adminClient(), index, id, fields); + } + + protected static void index(RestClient client, String index, String id, Object... fields) throws IOException { + XContentBuilder document = jsonBuilder().startObject(); + for (int i = 0; i < fields.length; i += 2) { + document.field((String) fields[i], fields[i + 1]); + } + document.endObject(); + final Request request = new Request("POST", "/" + index + "/_doc" + (id == null ? "" : "/" + id)); + request.setJsonEntity(Strings.toString(document)); + assertOK(client.performRequest(request)); + } + + protected static void resumeFollow(String followIndex) throws IOException { + final Request request = new Request("POST", "/" + followIndex + "/_ccr/resume_follow"); + request.setJsonEntity("{\"read_poll_timeout\": \"10ms\"}"); + assertOK(client().performRequest(request)); + } + + protected static void followIndex(String leaderIndex, String followIndex) throws IOException { + followIndex("leader_cluster", leaderIndex, followIndex); + } + + protected static void followIndex(String leaderCluster, String leaderIndex, String followIndex) throws IOException { + followIndex(client(), leaderCluster, leaderIndex, followIndex); + } + + protected static void followIndex(RestClient client, String leaderCluster, String leaderIndex, String followIndex) throws IOException { + followIndex(client, leaderCluster, leaderIndex, followIndex, null); + } + + protected static void followIndex( + final RestClient client, + final String leaderCluster, + final String leaderIndex, + final String followIndex, + final Settings settings + ) throws IOException { + final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow?wait_for_active_shards=1"); + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + { + bodyBuilder.field("remote_cluster", leaderCluster); + bodyBuilder.field("leader_index", leaderIndex); + bodyBuilder.field("read_poll_timeout", "10ms"); + if (settings != null) { + bodyBuilder.startObject("settings"); + { + settings.toXContent(bodyBuilder, ToXContent.EMPTY_PARAMS); + } + bodyBuilder.endObject(); + } + } + bodyBuilder.endObject(); + request.setJsonEntity(Strings.toString(bodyBuilder)); + } + assertOK(client.performRequest(request)); + } + + protected static void pauseFollow(String followIndex) throws IOException { + pauseFollow(client(), followIndex); + } + + protected static void pauseFollow(RestClient client, String followIndex) throws IOException { + assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow"))); + } + + protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException { + Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName); + putPatternRequest.setJsonEntity(String.format(Locale.ROOT, """ + {"leader_index_patterns": ["%s"], "remote_cluster": "%s"} + """, indexPattern, remoteCluster)); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void deleteAutoFollowPattern(String patternName) throws IOException { + deleteAutoFollowPattern(client(), patternName); + } + + protected static void deleteAutoFollowPattern(RestClient client, String patternName) throws IOException { + Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName); + assertOK(client.performRequest(putPatternRequest)); + } + + protected static void unfollow(String followIndex) throws IOException { + assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow"))); + } + + protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException { + verifyDocuments(index, expectedNumDocs, query, adminClient()); + } + + protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query, final RestClient client) + throws IOException { + final Request request = new Request("GET", "/" + index + "/_search"); + request.addParameter("size", Integer.toString(expectedNumDocs)); + request.addParameter("sort", "field:asc"); + request.addParameter("q", query); + request.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); + Map response = toMap(client.performRequest(request)); + + int numDocs = (int) XContentMapValues.extractValue("hits.total", response); + assertThat(index, numDocs, equalTo(expectedNumDocs)); + + List hits = (List) XContentMapValues.extractValue("hits.hits", response); + assertThat(hits.size(), equalTo(expectedNumDocs)); + for (int i = 0; i < expectedNumDocs; i++) { + int value = (int) XContentMapValues.extractValue("_source.field", (Map) hits.get(i)); + assertThat(index, i, equalTo(value)); + } + } + + protected static void verifyDocuments(final RestClient client, final String index, final int expectedNumDocs) throws IOException { + final Request request = new Request("GET", "/" + index + "/_search"); + request.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); + Map response = toMap(client.performRequest(request)); + + int numDocs = (int) XContentMapValues.extractValue("hits.total", response); + assertThat(index, numDocs, equalTo(expectedNumDocs)); + } + + protected static void verifyCcrMonitoring(final String expectedLeaderIndex, final String expectedFollowerIndex) throws IOException { + Request request = new Request("GET", "/.monitoring-*/_search"); + request.setJsonEntity(String.format(Locale.ROOT, """ + {"query": {"term": {"ccr_stats.leader_index": "%s"}}} + """, expectedLeaderIndex)); + Map response; + try { + response = toMap(adminClient().performRequest(request)); + } catch (ResponseException e) { + throw new AssertionError("error while searching", e); + } + + int followerMaxSeqNo = 0; + int followerMappingVersion = 0; + int followerSettingsVersion = 0; + int followerAliasesVersion = 0; + + List hits = (List) XContentMapValues.extractValue("hits.hits", response); + assertThat(hits.size(), greaterThanOrEqualTo(1)); + + for (int i = 0; i < hits.size(); i++) { + Map hit = (Map) hits.get(i); + String leaderIndex = (String) XContentMapValues.extractValue("_source.ccr_stats.leader_index", hit); + assertThat(leaderIndex, endsWith(expectedLeaderIndex)); + + final String followerIndex = (String) XContentMapValues.extractValue("_source.ccr_stats.follower_index", hit); + assertThat(followerIndex, equalTo(expectedFollowerIndex)); + + int foundFollowerMaxSeqNo = (int) XContentMapValues.extractValue("_source.ccr_stats.follower_max_seq_no", hit); + followerMaxSeqNo = Math.max(followerMaxSeqNo, foundFollowerMaxSeqNo); + int foundFollowerMappingVersion = (int) XContentMapValues.extractValue("_source.ccr_stats.follower_mapping_version", hit); + followerMappingVersion = Math.max(followerMappingVersion, foundFollowerMappingVersion); + int foundFollowerSettingsVersion = (int) XContentMapValues.extractValue("_source.ccr_stats.follower_settings_version", hit); + followerSettingsVersion = Math.max(followerSettingsVersion, foundFollowerSettingsVersion); + int foundFollowerAliasesVersion = (int) XContentMapValues.extractValue("_source.ccr_stats.follower_aliases_version", hit); + followerAliasesVersion = Math.max(followerAliasesVersion, foundFollowerAliasesVersion); + } + + assertThat(followerMaxSeqNo, greaterThan(0)); + assertThat(followerMappingVersion, greaterThan(0)); + assertThat(followerSettingsVersion, greaterThan(0)); + assertThat(followerAliasesVersion, greaterThan(0)); + } + + protected static void verifyAutoFollowMonitoring() throws IOException { + Request request = new Request("GET", "/.monitoring-*/_count"); + request.setJsonEntity(""" + { + "query": { + "bool" : { + "filter": { + "term" : { "type" : "ccr_auto_follow_stats" } + }, + "must" : { + "range" : { + "ccr_auto_follow_stats.number_of_successful_follow_indices" : { "gt" : 0 } + } + } + } + } + } + """); + String responseEntity; + Map response; + try { + responseEntity = EntityUtils.toString(adminClient().performRequest(request).getEntity()); + response = toMap(responseEntity); + } catch (ResponseException e) { + throw new AssertionError("error while searching", e); + } + assertNotNull(responseEntity); + + final Number count = (Number) XContentMapValues.extractValue("count", response); + assertThat( + "Expected at least 1 successfully followed index but found none, count returned [" + responseEntity + ']', + count.longValue(), + greaterThanOrEqualTo(1L) + ); + } + + protected static Map toMap(Response response) throws IOException { + return toMap(EntityUtils.toString(response.getEntity())); + } + + protected static Map toMap(String response) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false); + } + + protected static void ensureYellow(final String index) throws IOException { + ensureYellow(index, adminClient()); + } + + protected static void ensureYellow(final String index, final RestClient client) throws IOException { + ensureHealth(client, index, request -> { + request.addParameter("wait_for_status", "yellow"); + request.addParameter("wait_for_active_shards", "1"); + request.addParameter("wait_for_no_relocating_shards", "true"); + // follower index can be yellow even when its primary shards are still initializing as we bootstrap them using snapshot/restore. + request.addParameter("wait_for_no_initializing_shards", "true"); + request.addParameter("timeout", "5s"); + request.addParameter("level", "shards"); + }); + } + + protected Set getCcrNodeTasks() throws IOException { + final Request request = new Request("GET", "/_tasks"); + request.addParameter("detailed", "true"); + Map rsp1 = toMap(adminClient().performRequest(request)); + Map nodes = (Map) rsp1.get("nodes"); + assertThat(nodes.size(), equalTo(1)); + Map node = (Map) nodes.values().iterator().next(); + Map nodeTasks = (Map) node.get("tasks"); + var ccrNodeTasks = new HashSet(); + for (Map.Entry entry : nodeTasks.entrySet()) { + Map nodeTask = (Map) entry.getValue(); + String action = (String) nodeTask.get("action"); + if (action.startsWith("xpack/ccr/shard_follow_task")) { + var status = (Map) nodeTask.get("status"); + ccrNodeTasks.add( + new CcrNodeTask( + (String) status.get("remote_cluster"), + (String) status.get("leader_index"), + (String) status.get("follower_index"), + (Integer) status.get("shard_id") + ) + ); + } + } + return ccrNodeTasks; + } + + protected record CcrNodeTask(String remoteCluster, String leaderIndex, String followerIndex, int shardId) {} + + protected static boolean indexExists(String index) throws IOException { + Response response = adminClient().performRequest(new Request("HEAD", "/" + index)); + return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); + } + + protected static List verifyDataStream(final RestClient client, final String name, final String... expectedBackingIndices) + throws IOException { + Request request = new Request("GET", "/_data_stream/" + name); + Map response = toMap(client.performRequest(request)); + List retrievedDataStreams = (List) response.get("data_streams"); + assertThat(retrievedDataStreams, hasSize(1)); + List actualBackingIndexItems = (List) ((Map) retrievedDataStreams.get(0)).get("indices"); + assertThat(actualBackingIndexItems, hasSize(expectedBackingIndices.length)); + final List actualBackingIndices = new ArrayList<>(); + for (int i = 0; i < expectedBackingIndices.length; i++) { + Map actualBackingIndexItem = (Map) actualBackingIndexItems.get(i); + String actualBackingIndex = (String) actualBackingIndexItem.get("index_name"); + String expectedBackingIndex = expectedBackingIndices[i]; + + String actualDataStreamName = actualBackingIndex.substring(5, actualBackingIndex.indexOf('-', 5)); + String expectedDataStreamName = expectedBackingIndex.substring(5, expectedBackingIndex.indexOf('-', 5)); + assertThat(actualDataStreamName, equalTo(expectedDataStreamName)); + + int actualGeneration = Integer.parseInt(actualBackingIndex.substring(actualBackingIndex.lastIndexOf('-'))); + int expectedGeneration = Integer.parseInt(expectedBackingIndex.substring(expectedBackingIndex.lastIndexOf('-'))); + assertThat(actualGeneration, equalTo(expectedGeneration)); + actualBackingIndices.add(actualBackingIndex); + } + return List.copyOf(actualBackingIndices); + } + + protected static void createAutoFollowPattern( + RestClient client, + String name, + String pattern, + String remoteCluster, + String followIndexPattern + ) throws IOException { + Request request = new Request("PUT", "/_ccr/auto_follow/" + name); + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + { + bodyBuilder.array("leader_index_patterns", pattern); + if (followIndexPattern != null) { + bodyBuilder.field("follow_index_pattern", followIndexPattern); + } + bodyBuilder.field("remote_cluster", remoteCluster); + } + bodyBuilder.endObject(); + request.setJsonEntity(Strings.toString(bodyBuilder)); + } + assertOK(client.performRequest(request)); + } + + /** + * Fix point in time when data stream backing index is first time queried. + * This is required to avoid failures when running test at midnight. + * (index is created for day0, but assertions are executed for day1 assuming different time based index name that does not exist) + */ + private final LazyInitializable time = new LazyInitializable<>(System::currentTimeMillis); + + protected String backingIndexName(String dataStreamName, int generation) { + return DataStream.getDefaultBackingIndexName(dataStreamName, generation, time.getOrCompute()); + } + + protected RestClient buildLeaderClient() throws IOException { + assert targetCluster != TargetCluster.LEADER; + return buildClient(getLeaderCluster().getHttpAddresses()); + } + + protected RestClient buildLeaderClient(final Settings settings) throws IOException { + assert targetCluster != TargetCluster.LEADER; + return buildClient(getLeaderCluster().getHttpAddresses(), settings); + } + + protected RestClient buildMiddleClient() throws IOException { + assert targetCluster != TargetCluster.MIDDLE; + return buildClient(getMiddleCluster().getHttpAddresses()); + } + + private RestClient buildClient(final String url) throws IOException { + return buildClient(url, restAdminSettings()); + } + + private RestClient buildClient(final String url, final Settings settings) throws IOException { + int portSeparator = url.lastIndexOf(':'); + HttpHost httpHost = new HttpHost( + url.substring(0, portSeparator), + Integer.parseInt(url.substring(portSeparator + 1)), + getProtocol() + ); + return buildClient(settings, new HttpHost[] { httpHost }); + } + + public enum TargetCluster { + LEADER, + MIDDLE, + FOLLOWER; + } + + public static class TargetClusterTestOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + return Integer.compare(getOrdinal(o1), getOrdinal(o2)); + } + + private int getOrdinal(TestMethodAndParams t) { + return ((TargetCluster) t.getInstanceArguments().get(0)).ordinal(); + } + } +} diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java similarity index 94% rename from x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java rename to x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index 6f66e7e386066..bc3c08fdb9acf 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -7,6 +7,10 @@ package org.elasticsearch.xpack.ccr; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.SuppressForbidden; + import org.apache.http.client.methods.HttpPost; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; @@ -20,9 +24,15 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.text.SimpleDateFormat; @@ -36,7 +46,6 @@ import static org.elasticsearch.xcontent.ObjectPath.eval; import static org.elasticsearch.xpack.core.ilm.ShrinkIndexNameSupplier.SHRUNKEN_INDEX_PREFIX; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasEntry; @@ -47,12 +56,78 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; -public class AutoFollowIT extends ESCCRRestTestCase { +@SuppressForbidden("temp folder uses file api") +public class AutoFollowIT extends AbstractCCRRestTestCase { private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss", Locale.ROOT); + public static TemporaryFolder leaderRepoDir = new TemporaryFolder(); + + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .module("searchable-snapshots") + .module("data-streams") + .module("ingest-common") + .module("mapper-extras") + .module("x-pack-stack") + .module("x-pack-ilm") + .module("x-pack-monitoring") + .module("constant-keyword") + .module("wildcard") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .setting("path.repo", () -> leaderRepoDir.getRoot().getAbsolutePath()) + .feature(FeatureFlag.TIME_SERIES_MODE) + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster middleCluster = ElasticsearchCluster.local() + .name("middle-cluster") + .apply(commonConfig) + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("xpack.monitoring.collection.enabled", "true") + .setting("cluster.remote.middle_cluster.seeds", () -> "\"" + middleCluster.getTransportEndpoints() + "\"") + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderRepoDir) + .around(leaderCluster) + .around(middleCluster) + .around(followerCluster); + + public AutoFollowIT(@Name("targetCluster") AbstractCCRRestTestCase.TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderMiddleFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } + + @Override + protected ElasticsearchCluster getMiddleCluster() { + return middleCluster; + } + public void testMultipleAutoFollowPatternsDifferentClusters() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { logger.info("skipping test, waiting for target cluster [follow]"); return; } @@ -108,7 +183,7 @@ public void testMultipleAutoFollowPatternsDifferentClusters() throws Exception { } public void testAutoFollowPatterns() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { logger.info("skipping test, waiting for target cluster [follow]"); return; } @@ -178,7 +253,7 @@ public void testAutoFollowPatterns() throws Exception { }); assertLongBusy(() -> verifyCcrMonitoring("metrics-20210101", "metrics-20210101")); - assertLongBusy(ESCCRRestTestCase::verifyAutoFollowMonitoring); + assertLongBusy(AbstractCCRRestTestCase::verifyAutoFollowMonitoring); } finally { cleanUpFollower(List.of("metrics-20210101"), List.of(), List.of(autoFollowPatternName)); @@ -187,7 +262,7 @@ public void testAutoFollowPatterns() throws Exception { } public void testPutAutoFollowPatternThatOverridesRequiredLeaderSetting() throws IOException { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { logger.info("skipping test, waiting for target cluster [follow]"); return; } @@ -227,7 +302,7 @@ public void testPutAutoFollowPatternThatOverridesRequiredLeaderSetting() throws } public void testDataStreams() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -323,7 +398,7 @@ public void testDataStreams() throws Exception { } public void testDataStreamsRenameFollowDataStream() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -438,7 +513,7 @@ public void testDataStreamsRenameFollowDataStream() throws Exception { } public void testDataStreams_autoFollowAfterDataStreamCreated() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -516,7 +591,7 @@ public void testDataStreams_autoFollowAfterDataStreamCreated() throws Exception @SuppressWarnings("unchecked") public void testDataStreamsBackingIndicesOrdering() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -642,7 +717,7 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { } public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -765,7 +840,7 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { } public void testRolloverAliasInFollowClusterForbidden() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -849,7 +924,7 @@ private static void verifyAlias(RestClient client, String aliasName, boolean che } public void testDataStreamsBiDirectionalReplication() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -1034,7 +1109,7 @@ public void testDataStreamsBiDirectionalReplication() throws Exception { } public void testAutoFollowSearchableSnapshotsFails() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } @@ -1046,13 +1121,7 @@ public void testAutoFollowSearchableSnapshotsFails() throws Exception { final String indexName = testPrefix + "-index"; try { try (var leaderClient = buildLeaderClient()) { - final String systemPropertyRepoPath = System.getProperty("tests.leader_cluster_repository_path"); - assertThat( - "Missing system property [tests.leader_cluster_repository_path]", - systemPropertyRepoPath, - not(emptyOrNullString()) - ); - final String repositoryPath = systemPropertyRepoPath + '/' + testPrefix; + final String repositoryPath = leaderRepoDir.newFolder(testPrefix).getAbsolutePath(); registerRepository(leaderClient, repository, "fs", true, Settings.builder().put("location", repositoryPath).build()); @@ -1217,4 +1286,9 @@ private void cleanUp( } } } + + @Override + public String getTestName() { + return super.getTestName().replaceAll("[ ={}]", "_"); + } } diff --git a/x-pack/plugin/ccr/qa/non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java similarity index 55% rename from x-pack/plugin/ccr/qa/non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java rename to x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java index 2f5fabd21cf98..4afaa57a4a8cf 100644 --- a/x-pack/plugin/ccr/qa/non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java @@ -7,21 +7,64 @@ package org.elasticsearch.xpack.ccr; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; import java.util.Locale; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasToString; -public class CcrMultiClusterLicenseIT extends ESCCRRestTestCase { +public class CcrMultiClusterLicenseIT extends AbstractCCRRestTestCase { + + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .setting("xpack.security.enabled", "true") + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderCluster).around(followerCluster); + + public CcrMultiClusterLicenseIT(@Name("targetCluster") TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } public void testFollow() { - if ("follow".equals(targetCluster)) { + if (targetCluster == TargetCluster.FOLLOWER) { final Request request = new Request("PUT", "/follower/_ccr/follow"); request.setJsonEntity(""" {"remote_cluster": "leader_cluster", "leader_index": "leader"} @@ -31,7 +74,7 @@ public void testFollow() { } public void testAutoFollow() { - if ("follow".equals(targetCluster)) { + if (targetCluster == TargetCluster.FOLLOWER) { final Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern"); request.setJsonEntity(""" {"leader_index_patterns":["*"], "remote_cluster": "leader_cluster"} diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/ChainIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/ChainIT.java new file mode 100644 index 0000000000000..7f38c07b47340 --- /dev/null +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/ChainIT.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ccr; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; + +public class ChainIT extends AbstractCCRRestTestCase { + + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .feature(FeatureFlag.TIME_SERIES_MODE) + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster middleCluster = ElasticsearchCluster.local() + .name("middle-cluster") + .apply(commonConfig) + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("cluster.remote.middle_cluster.seeds", () -> "\"" + middleCluster.getTransportEndpoints() + "\"") + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderCluster).around(middleCluster).around(followerCluster); + + public ChainIT(@Name("targetCluster") AbstractCCRRestTestCase.TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderMiddleFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } + + @Override + protected ElasticsearchCluster getMiddleCluster() { + return middleCluster; + } + + public void testFollowIndex() throws Exception { + final int numDocs = 128; + final String leaderIndexName = "leader"; + final String middleIndexName = "middle"; + switch (targetCluster) { + case LEADER: + logger.info("Running against leader cluster"); + String mapping = ""; + if (randomBoolean()) { // randomly do source filtering on indexing + mapping = """ + "_source": { "includes": ["field"], "excludes": ["filtered_field"]}"""; + } + createIndex(adminClient(), leaderIndexName, Settings.EMPTY, mapping, null); + for (int i = 0; i < numDocs; i++) { + logger.info("Indexing doc [{}]", i); + index(client(), leaderIndexName, Integer.toString(i), "field", i, "filtered_field", "true"); + } + refresh(adminClient(), leaderIndexName); + verifyDocuments(leaderIndexName, numDocs, "filtered_field:true"); + break; + case MIDDLE: + logger.info("Running against middle cluster"); + followIndex("leader_cluster", leaderIndexName, middleIndexName); + assertBusy(() -> verifyDocuments(middleIndexName, numDocs, "filtered_field:true")); + try (RestClient leaderClient = buildLeaderClient()) { + int id = numDocs; + index(leaderClient, leaderIndexName, Integer.toString(id), "field", id, "filtered_field", "true"); + index(leaderClient, leaderIndexName, Integer.toString(id + 1), "field", id + 1, "filtered_field", "true"); + index(leaderClient, leaderIndexName, Integer.toString(id + 2), "field", id + 2, "filtered_field", "true"); + } + assertBusy(() -> verifyDocuments(middleIndexName, numDocs + 3, "filtered_field:true")); + break; + case FOLLOWER: + logger.info("Running against follow cluster"); + final String followIndexName = "follow"; + followIndex("middle_cluster", middleIndexName, followIndexName); + assertBusy(() -> verifyDocuments(followIndexName, numDocs + 3, "filtered_field:true")); + + try (RestClient leaderClient = buildLeaderClient()) { + int id = numDocs + 3; + index(leaderClient, leaderIndexName, Integer.toString(id), "field", id, "filtered_field", "true"); + index(leaderClient, leaderIndexName, Integer.toString(id + 1), "field", id + 1, "filtered_field", "true"); + index(leaderClient, leaderIndexName, Integer.toString(id + 2), "field", id + 2, "filtered_field", "true"); + } + + try (RestClient middleClient = buildMiddleClient()) { + assertBusy(() -> verifyDocuments(middleIndexName, numDocs + 6, "filtered_field:true", middleClient)); + } + + assertBusy(() -> verifyDocuments(followIndexName, numDocs + 6, "filtered_field:true")); + break; + } + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + +} diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java similarity index 70% rename from x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java rename to x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java index 457a0d4ad3f81..930f546e4f681 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java @@ -6,7 +6,9 @@ */ package org.elasticsearch.xpack.ccr; -import org.apache.lucene.util.Constants; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -15,13 +17,20 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.PathUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.LogType; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; import org.hamcrest.Matchers; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import java.io.BufferedReader; import java.io.IOException; -import java.nio.file.Path; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -32,12 +41,46 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.core.Is.is; -public class FollowIndexIT extends ESCCRRestTestCase { +public class DowngradeLicenseFollowIndexIT extends AbstractCCRRestTestCase { + + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "true") + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderCluster).around(followerCluster); + + public DowngradeLicenseFollowIndexIT(@Name("targetCluster") TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } public void testDowngradeRemoteClusterToBasic() throws Exception { - if ("follow".equals(targetCluster) == false) { - return; - } + assumeTrue("Test should only run with target_cluster=follow", targetCluster == TargetCluster.FOLLOWER); { Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern"); @@ -87,15 +130,15 @@ public void testDowngradeRemoteClusterToBasic() throws Exception { assertThat(indexExists(index2), is(false)); // parse the logs and ensure that the auto-coordinator skipped coordination on the leader cluster - // (does not work on windows...) - if (Constants.WINDOWS == false) { - assertBusy(() -> { - Path path = PathUtils.get(System.getProperty("log")); - try (Stream stream = JsonLogsStream.from(path)) { - assertTrue(stream.anyMatch(autoFollowCoordinatorWarn()::matches)); - } - }); - } + assertBusy(() -> { + try ( + InputStream in = followerCluster.getNodeLog(0, LogType.SERVER_JSON); + BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + Stream stream = JsonLogsStream.from(reader) + ) { + assertTrue(stream.anyMatch(autoFollowCoordinatorWarn()::matches)); + } + }); }, 60, TimeUnit.SECONDS); // Manually following index2 also does not work after the downgrade: diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java similarity index 85% rename from x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java rename to x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index e8e19bad2a7ef..1f89d316a4e3d 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -6,6 +6,10 @@ */ package org.elasticsearch.xpack.ccr; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.SuppressForbidden; + import org.apache.http.client.methods.HttpPost; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -21,28 +25,82 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; import java.io.IOException; -import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.not; -public class FollowIndexIT extends ESCCRRestTestCase { +@SuppressForbidden("temp folder uses file api") +public class FollowIndexIT extends AbstractCCRRestTestCase { + + public static TemporaryFolder leaderRepoDir = new TemporaryFolder(); + + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .module("searchable-snapshots") + .module("data-streams") + .module("ingest-common") + .module("mapper-extras") + .module("x-pack-stack") + .module("x-pack-ilm") + .module("x-pack-monitoring") + .module("constant-keyword") + .module("wildcard") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .setting("path.repo", () -> leaderRepoDir.getRoot().getAbsolutePath()) + .feature(FeatureFlag.TIME_SERIES_MODE) + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("xpack.monitoring.collection.enabled", "true") + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderRepoDir).around(leaderCluster).around(followerCluster); + + public FollowIndexIT(@Name("targetCluster") TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } public void testFollowIndex() throws Exception { final int numDocs = 128; final String leaderIndexName = "test_index1"; - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { logger.info("Running against leader cluster"); String mapping = ""; if (randomBoolean()) { // randomly do source filtering on indexing @@ -56,7 +114,7 @@ public void testFollowIndex() throws Exception { } refresh(adminClient(), leaderIndexName); verifyDocuments(leaderIndexName, numDocs, "filtered_field:true"); - } else if ("follow".equals(targetCluster)) { + } else if (targetCluster == TargetCluster.FOLLOWER) { logger.info("Running against follow cluster"); final String followIndexName = "test_index2"; final boolean overrideNumberOfReplicas = randomBoolean(); @@ -100,7 +158,7 @@ public void testFollowIndex() throws Exception { } public void testFollowThatOverridesRequiredLeaderSetting() throws IOException { - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { createIndex(adminClient(), "override_leader_index", Settings.EMPTY); } else { final Settings settings = Settings.builder().put("index.number_of_shards", 5).build(); @@ -124,7 +182,7 @@ public void testFollowThatOverridesRequiredLeaderSetting() throws IOException { } public void testFollowThatOverridesNonExistentSetting() throws IOException { - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { createIndex(adminClient(), "override_leader_index_non_existent_setting", Settings.EMPTY); } else { final Settings settings = Settings.builder().put("index.non_existent_setting", randomAlphaOfLength(3)).build(); @@ -151,7 +209,7 @@ public void testFollowThatOverridesNonExistentSetting() throws IOException { } public void testFollowNonExistingLeaderIndex() { - if ("follow".equals(targetCluster) == false) { + if (targetCluster == TargetCluster.FOLLOWER == false) { logger.info("skipping test, waiting for target cluster [follow]"); return; } @@ -165,7 +223,7 @@ public void testFollowNonExistingLeaderIndex() { } public void testFollowDataStreamFails() throws Exception { - if ("follow".equals(targetCluster) == false) { + if (targetCluster == TargetCluster.FOLLOWER == false) { return; } @@ -182,13 +240,11 @@ public void testFollowDataStreamFails() throws Exception { } public void testFollowSearchableSnapshotsFails() throws Exception { - final String testPrefix = getTestName().toLowerCase(Locale.ROOT); + final String testPrefix = "test_follow_searchable_snapshots_fails"; final String mountedIndex = "mounted-" + testPrefix; - if ("leader".equals(targetCluster)) { - final String systemPropertyRepoPath = System.getProperty("tests.leader_cluster_repository_path"); - assertThat("Missing system property [tests.leader_cluster_repository_path]", systemPropertyRepoPath, not(emptyOrNullString())); - final String repositoryPath = systemPropertyRepoPath + '/' + testPrefix; + if (targetCluster == TargetCluster.LEADER) { + final String repositoryPath = leaderRepoDir.newFolder(testPrefix).getAbsolutePath(); final String repository = "repository-" + testPrefix; registerRepository(repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build()); @@ -227,7 +283,7 @@ public void testFollowTsdbIndex() throws Exception { final int numDocs = 128; final String leaderIndexName = "tsdb_leader"; long basetime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2021-04-28T18:35:24.467Z"); - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { logger.info("Running against leader cluster"); createIndex( adminClient(), @@ -248,7 +304,7 @@ public void testFollowTsdbIndex() throws Exception { } refresh(adminClient(), leaderIndexName); verifyDocuments(client(), leaderIndexName, numDocs); - } else if ("follow".equals(targetCluster)) { + } else if (targetCluster == TargetCluster.FOLLOWER) { logger.info("Running against follow cluster"); final String followIndexName = "tsdb_follower"; final boolean overrideNumberOfReplicas = randomBoolean(); @@ -321,7 +377,7 @@ public void testFollowTsdbIndex() throws Exception { } public void testFollowTsdbIndexCanNotOverrideMode() throws Exception { - if (false == "follow".equals(targetCluster)) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } logger.info("Running against follow cluster"); @@ -342,7 +398,7 @@ public void testFollowTsdbIndexCanNotOverrideMode() throws Exception { } public void testFollowStandardIndexCanNotOverrideMode() throws Exception { - if (false == "follow".equals(targetCluster)) { + if (targetCluster != TargetCluster.FOLLOWER) { return; } logger.info("Running against follow cluster"); @@ -365,7 +421,7 @@ public void testFollowStandardIndexCanNotOverrideMode() throws Exception { public void testSyntheticSource() throws Exception { final int numDocs = 128; final String leaderIndexName = "synthetic_leader"; - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { logger.info("Running against leader cluster"); Settings settings = Settings.builder() .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) @@ -378,7 +434,7 @@ public void testSyntheticSource() throws Exception { } refresh(adminClient(), leaderIndexName); verifyDocuments(client(), leaderIndexName, numDocs); - } else if ("follow".equals(targetCluster)) { + } else if (targetCluster == TargetCluster.FOLLOWER) { logger.info("Running against follow cluster"); final String followIndexName = "synthetic_follower"; final boolean overrideNumberOfReplicas = randomBoolean(); diff --git a/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java similarity index 85% rename from x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java rename to x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java index e6c68b0580374..0adf0b31b4ebd 100644 --- a/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java @@ -6,6 +6,9 @@ */ package org.elasticsearch.xpack.ccr; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -21,7 +24,12 @@ import org.elasticsearch.core.Strings; import org.elasticsearch.health.node.selection.HealthNode; import org.elasticsearch.index.seqno.ReplicationTracker; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ObjectPath; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; import java.io.IOException; import java.text.SimpleDateFormat; @@ -39,7 +47,57 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -public class FollowIndexSecurityIT extends ESCCRRestTestCase { +public class FollowIndexSecurityIT extends AbstractCCRRestTestCase { + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .module("mapper-extras") + .module("data-streams") + .module("ingest-common") + .module("x-pack-monitoring") + .module("x-pack-ilm") + .module("wildcard") + .module("x-pack-stack") + .module("constant-keyword") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user("test_admin", "x-pack-test-password", "superuser", false) + .user("test_ccr", "x-pack-test-password", "ccruser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local() + .name("leader-cluster") + .apply(commonConfig) + .rolesFile(Resource.fromClasspath("leader-roles.yml")) + .build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("xpack.monitoring.collection.enabled", "false") + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .rolesFile(Resource.fromClasspath("follower-roles.yml")) + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderCluster).around(followerCluster); + + public FollowIndexSecurityIT(@Name("targetCluster") TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } @Override protected Settings restClientSettings() { @@ -57,7 +115,7 @@ public void testFollowIndex() throws Exception { final int numDocs = 16; final String allowedIndex = "allowed-index"; final String unallowedIndex = "unallowed-index"; - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { logger.info("Running against leader cluster"); createIndex(adminClient(), allowedIndex, Settings.EMPTY); createIndex(adminClient(), unallowedIndex, Settings.EMPTY); @@ -147,9 +205,8 @@ public void testFollowIndex() throws Exception { } public void testAutoFollowPatterns() throws Exception { - assumeTrue("Test should only run with target_cluster=follow", "follow".equals(targetCluster)); - - final String prefix = getTestName().toLowerCase(Locale.ROOT); + assumeTrue("Test should only run with target_cluster=follow", targetCluster == TargetCluster.FOLLOWER); + final String prefix = "testautofollowpatterns"; String allowedIndex = prefix + "-eu_20190101"; String disallowedIndex = prefix + "-us_20190101"; @@ -188,7 +245,7 @@ public void testAutoFollowPatterns() throws Exception { assertThat(indexExists(disallowedIndex), is(false)); withMonitoring(logger, () -> { assertBusy(() -> verifyCcrMonitoring(allowedIndex, allowedIndex), 120L, TimeUnit.SECONDS); - assertBusy(ESCCRRestTestCase::verifyAutoFollowMonitoring, 120L, TimeUnit.SECONDS); + assertBusy(AbstractCCRRestTestCase::verifyAutoFollowMonitoring, 120L, TimeUnit.SECONDS); }); } finally { // Cleanup by deleting auto follow pattern and pause following: @@ -204,7 +261,7 @@ public void testAutoFollowPatterns() throws Exception { public void testForgetFollower() throws IOException { final String forgetLeader = "forget-leader"; final String forgetFollower = "forget-follower"; - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { logger.info("running against leader cluster"); createIndex(adminClient(), forgetLeader, indexSettings(1, 0).build()); } else { @@ -253,7 +310,7 @@ public void testForgetFollower() throws IOException { public void testCleanShardFollowTaskAfterDeleteFollower() throws Exception { final String cleanLeader = "clean-leader"; final String cleanFollower = "clean-follower"; - if ("leader".equals(targetCluster)) { + if (targetCluster == TargetCluster.LEADER) { logger.info("running against leader cluster"); final Settings indexSettings = indexSettings(1, 0).put("index.soft_deletes.enabled", true).build(); createIndex(adminClient(), cleanLeader, indexSettings); @@ -270,9 +327,11 @@ public void testCleanShardFollowTaskAfterDeleteFollower() throws Exception { } public void testUnPromoteAndFollowDataStream() throws Exception { - assumeTrue("Test should only run with target_cluster=follow", "follow".equals(targetCluster)); + assumeTrue("Test should only run with target_cluster=follow", targetCluster == TargetCluster.FOLLOWER); var numDocs = 64; + // TODO: We're implicitly relying on index templates from the stack module here. This requires us to install this module + // and several others. We should think about just explicitly creating a data stream index template instead. var dataStreamName = "logs-eu-monitor1"; var dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss", Locale.ROOT); diff --git a/x-pack/plugin/ccr/qa/restart/src/test/java/org/elasticsearch/xpack/ccr/RestartIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/RestartIT.java similarity index 71% rename from x-pack/plugin/ccr/qa/restart/src/test/java/org/elasticsearch/xpack/ccr/RestartIT.java rename to x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/RestartIT.java index c1c5cda1b6597..dfe1c8f6ce816 100644 --- a/x-pack/plugin/ccr/qa/restart/src/test/java/org/elasticsearch/xpack/ccr/RestartIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/RestartIT.java @@ -7,26 +7,68 @@ package org.elasticsearch.xpack.ccr; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; import java.io.IOException; -public class RestartIT extends ESCCRRestTestCase { +public class RestartIT extends AbstractCCRRestTestCase { + + public static LocalClusterConfigProvider commonConfig = c -> c.module("x-pack-ccr") + .module("analysis-common") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster leaderCluster = ElasticsearchCluster.local().name("leader-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster followerCluster = ElasticsearchCluster.local() + .name("follow-cluster") + .apply(commonConfig) + .setting("cluster.remote.leader_cluster.seeds", () -> "\"" + leaderCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(leaderCluster).around(followerCluster); + + public RestartIT(@Name("targetCluster") TargetCluster targetCluster) { + super(targetCluster); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return leaderFollower(); + } + + @Override + protected ElasticsearchCluster getLeaderCluster() { + return leaderCluster; + } + + @Override + protected ElasticsearchCluster getFollowerCluster() { + return followerCluster; + } public void testRestart() throws Exception { final int numberOfDocuments = 128; - final String testsTargetCluster = System.getProperty("tests.target_cluster"); - switch (testsTargetCluster) { - case "leader" -> { + switch (targetCluster) { + case LEADER -> { // create a single index "leader" on the leader createIndexAndIndexDocuments("leader", numberOfDocuments, client()); } - case "follow" -> { + case FOLLOWER -> { // follow "leader" with "follow-leader" on the follower followIndex("leader", "follow-leader"); verifyFollower("follow-leader", numberOfDocuments, client()); @@ -48,8 +90,11 @@ public void testRestart() throws Exception { // the follower should catch up verifyFollower("follow-leader-1", numberOfDocuments, client()); } - } - case "follow-restart" -> { + + followerCluster.restart(false); + closeClients(); + initClient(); + try (RestClient leaderClient = buildLeaderClient()) { // create "leader-2" on the leader, and index some additional documents into existing indices createIndexAndIndexDocuments("leader-2", numberOfDocuments, leaderClient); @@ -68,7 +113,7 @@ public void testRestart() throws Exception { } } default -> { - throw new IllegalArgumentException("unexpected value [" + testsTargetCluster + "] for tests.target_cluster"); + throw new IllegalArgumentException("unexpected value [" + targetCluster + "] for targetCluster"); } } } diff --git a/x-pack/plugin/ccr/qa/security/follower-roles.yml b/x-pack/plugin/ccr/src/javaRestTest/resources/follower-roles.yml similarity index 100% rename from x-pack/plugin/ccr/qa/security/follower-roles.yml rename to x-pack/plugin/ccr/src/javaRestTest/resources/follower-roles.yml diff --git a/x-pack/plugin/ccr/qa/security/leader-roles.yml b/x-pack/plugin/ccr/src/javaRestTest/resources/leader-roles.yml similarity index 100% rename from x-pack/plugin/ccr/qa/security/leader-roles.yml rename to x-pack/plugin/ccr/src/javaRestTest/resources/leader-roles.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/ccr/CcrRestIT.java b/x-pack/plugin/ccr/src/yamlRestTest/java/org/elasticsearch/xpack/ccr/CcrRestIT.java similarity index 59% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/ccr/CcrRestIT.java rename to x-pack/plugin/ccr/src/yamlRestTest/java/org/elasticsearch/xpack/ccr/CcrRestIT.java index b22941ab0c2a6..482bcc829eb50 100644 --- a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/ccr/CcrRestIT.java +++ b/x-pack/plugin/ccr/src/yamlRestTest/java/org/elasticsearch/xpack/ccr/CcrRestIT.java @@ -12,13 +12,30 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; public class CcrRestIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + // TODO: Switch to ingeg-test when we fix the xpack info api + .distribution(DistributionType.DEFAULT) + .module("x-pack-ccr") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user("ccr-user", "ccr-user-password", "superuser", false) + // Disable assertions in FollowingEngineAssertions, otherwise an AssertionError is thrown before + // indexing a document directly in a follower index. In a rest test we like to test the exception + // that is thrown in production when indexing a document directly in a follower index. + .jvmArg("-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions") + .build(); + public CcrRestIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -44,4 +61,8 @@ public void cleanup() throws Exception { waitForPendingTasks(adminClient(), taskName -> taskName.startsWith("indices:data/read/xpack/ccr/shard_changes")); } + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_info.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_info.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_info.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_info.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_stats.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_stats.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_stats.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/follow_stats.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/forget_follower.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/stats.yml b/x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/stats.yml similarity index 100% rename from x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/stats.yml rename to x-pack/plugin/ccr/src/yamlRestTest/resources/rest-api-spec/test/ccr/stats.yml diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 0dfd9e206c5d4..4987bc6bd214e 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -7,9 +7,9 @@ import java.nio.file.Paths apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-java-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-test-artifact' base { @@ -94,12 +94,12 @@ tasks.named("processResources").configure { String licenseKey = providers.systemProperty("license.key").getOrNull() if (licenseKey != null) { println "Using provided license key from ${licenseKey}" - } else if (buildParams.isSnapshotBuild()) { + } else if (buildParams.snapshotBuild) { licenseKey = Paths.get(project.projectDir.path, 'snapshot.key') } else { throw new IllegalArgumentException('Property license.key must be set for release build') } - File licenseKeyFile = rootProject.file(licenseKey) + File licenseKeyFile = layout.settingsDirectory.file(licenseKey).asFile if (licenseKeyFile.exists() == false) { throw new IllegalArgumentException('license.key at specified path [' + licenseKey + '] does not exist') } @@ -144,19 +144,11 @@ restResources { } } -testClusters.configureEach { - testDistribution = 'default' - setting 'xpack.security.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - //disabling ILM history as it disturbs testDSXpackUsage test - setting 'indices.lifecycle.history_index_enabled', 'false' - keystore 'bootstrap.password', 'x-pack-test-password' - user username: "x_pack_rest_user", password: "x-pack-test-password" - requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.15.0") - systemProperty 'es.queryable_built_in_roles_enabled', 'false' +tasks.named("javaRestTest") { + usesDefaultDistribution("uses the _xpack api") } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.withType(Test).configureEach { systemProperty 'es.failure_store_feature_flag_enabled', 'true' } diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java index 600a41ed01beb..6812d6179cb5e 100644 --- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java +++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java @@ -13,10 +13,14 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; import java.util.List; import java.util.Map; @@ -27,6 +31,18 @@ public class DataStreamRestIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .setting("indices.lifecycle.history_index_enabled", "false") + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password") + .systemProperty("es.queryable_built_in_roles_enabled", "false") + .build(); + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password")); @Override @@ -143,4 +159,9 @@ private void putFailureStoreTemplate() { throw new RuntimeException(e); } } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/LicenseInstallationIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/LicenseInstallationIT.java index 1f670e0ee2d8c..aa95b1a528f12 100644 --- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/LicenseInstallationIT.java +++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/LicenseInstallationIT.java @@ -18,11 +18,14 @@ import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.license.TestUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; +import org.junit.ClassRule; import java.io.IOException; import java.util.Locale; @@ -39,6 +42,16 @@ */ public class LicenseInstallationIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password") + .systemProperty("es.queryable_built_in_roles_enabled", "false") + .build(); + @Override protected Settings restClientSettings() { String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray())); @@ -159,4 +172,9 @@ private void assertClusterUsingTrialLicense() throws Exception { assertThat("the cluster should be using a trial license", innerMap.get("type"), equalTo("trial")); }); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/StackTemplatesRestIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/StackTemplatesRestIT.java index fcbf955c2b9ae..534f6bca07a65 100644 --- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/StackTemplatesRestIT.java +++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/StackTemplatesRestIT.java @@ -12,12 +12,27 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; import static org.hamcrest.Matchers.is; public class StackTemplatesRestIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password") + .systemProperty("es.queryable_built_in_roles_enabled", "false") + .build(); + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password")); @Override @@ -56,4 +71,9 @@ public void testTemplatesCanBeDisabled() throws Exception { ResponseException exception = expectThrows(ResponseException.class, () -> client.performRequest(deleteRequest)); assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(404)); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 3e9f49a7d01e4..701174d4c8599 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -115,3 +115,6 @@ logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter logger.samlxml_decrypt.level = fatal logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter logger.saml2_decrypt.level = fatal + +logger.entitlements_xpack_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security +logger.entitlements_xpack_security.level = error diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 55b3428907c40..5b00c22ed3372 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -7,6 +7,7 @@ module org.elasticsearch.xcore { requires org.elasticsearch.cli; + requires org.elasticsearch.entitlement; requires org.elasticsearch.base; requires org.elasticsearch.grok; requires org.elasticsearch.server; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackBuild.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackBuild.java deleted file mode 100644 index 1cea4c23bcca0..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackBuild.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core; - -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.core.SuppressForbidden; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.jar.JarInputStream; -import java.util.jar.Manifest; - -/** - * Information about the built version of x-pack that is running. - */ -public class XPackBuild { - - public static final XPackBuild CURRENT; - - static { - final String shortHash; - final String date; - - Path path = getElasticsearchCodebase(); - if (path.toString().endsWith(".jar")) { - try (JarInputStream jar = new JarInputStream(Files.newInputStream(path))) { - Manifest manifest = jar.getManifest(); - shortHash = manifest.getMainAttributes().getValue("Change"); - date = manifest.getMainAttributes().getValue("Build-Date"); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - // not running from a jar (unit tests, IDE) - shortHash = "Unknown"; - date = "Unknown"; - } - - CURRENT = new XPackBuild(shortHash, date); - } - - /** - * Returns path to xpack codebase path - */ - @SuppressForbidden(reason = "looks up path of xpack.jar directly") - static Path getElasticsearchCodebase() { - URL url = XPackBuild.class.getProtectionDomain().getCodeSource().getLocation(); - try { - return PathUtils.get(url.toURI()); - } catch (URISyntaxException bogus) { - throw new RuntimeException(bogus); - } - } - - private String shortHash; - private String date; - - XPackBuild(String shortHash, String date) { - this.shortHash = shortHash; - this.date = date; - } - - public String shortHash() { - return shortHash; - } - - public String date() { - return date; - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index f79a3fbf124b1..69eb17dee36a9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -412,9 +412,9 @@ public List getRestHandlers( } public static Path resolveConfigFile(Environment env, String name) { - Path config = env.configFile().resolve(name); + Path config = env.configDir().resolve(name); if (Files.exists(config) == false) { - Path legacyConfig = env.configFile().resolve("x-pack").resolve(name); + Path legacyConfig = env.configDir().resolve("x-pack").resolve(name); if (Files.exists(legacyConfig)) { deprecationLogger.warn( DeprecationCategory.OTHER, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 97927ded5ba49..f0b7e200e795f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -212,7 +212,7 @@ public Iterator> settings() { Property.NodeScope ); - private static final List JDK12_CIPHERS = List.of( + private static final List PRE_JDK24_CIPHERS = List.of( "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", // TLSv1.3 cipher has PFS, AEAD, hardware support "TLS_CHACHA20_POLY1305_SHA256", // TLSv1.3 cipher has PFS, AEAD @@ -238,7 +238,27 @@ public Iterator> settings() { "TLS_RSA_WITH_AES_128_CBC_SHA" ); // hardware support - public static final List DEFAULT_CIPHERS = JDK12_CIPHERS; + private static final List JDK24_CIPHERS = List.of( + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", // TLSv1.3 cipher has PFS, AEAD, hardware support + "TLS_CHACHA20_POLY1305_SHA256", // TLSv1.3 cipher has PFS, AEAD + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", // PFS, AEAD, hardware support + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", // PFS, AEAD, hardware support + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", // PFS, AEAD + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", // PFS, hardware support + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", // PFS, hardware support + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", // PFS, hardware support + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA" // PFS, hardware support + ); // hardware support + + public static final List DEFAULT_CIPHERS = Runtime.version().feature() < 24 ? PRE_JDK24_CIPHERS : JDK24_CIPHERS; public static final Setting PASSWORD_HASHING_ALGORITHM = defaultStoredPasswordHashAlgorithmSetting( "xpack.security.authc.password_hashing.algorithm", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java index 84e27b08c1d38..5a0c6a31d1fb8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.core.action; +import org.elasticsearch.Build; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; @@ -23,7 +24,6 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.XPackBuild; import java.util.HashSet; import java.util.List; @@ -58,7 +58,7 @@ protected void doExecute(Task task, XPackInfoRequest request, ActionListener T getTaskAndCheckAuthentication( TaskManager taskManager, AsyncExecutionId asyncExecutionId, Class tClass + ) throws IOException { + return getTaskAndCheckAuthentication(taskManager, security, asyncExecutionId, tClass); + } + + /** + * Returns the {@link AsyncTask} if the provided asyncTaskId + * is registered in the task manager, null otherwise. + * + * This method throws a {@link ResourceNotFoundException} if the authenticated user + * is not the creator of the original task. + */ + public static T getTaskAndCheckAuthentication( + TaskManager taskManager, + AsyncSearchSecurity security, + AsyncExecutionId asyncExecutionId, + Class tClass ) throws IOException { T asyncTask = getTask(taskManager, asyncExecutionId, tClass); if (asyncTask == null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java index c9624a8cc99b5..bfe40347e8ca5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditor.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; @@ -27,6 +28,7 @@ import java.util.Objects; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -50,6 +52,7 @@ public abstract class AbstractAuditor { private Queue backlog; private final AtomicBoolean indexAndAliasCreationInProgress; + private final ExecutorService executorService; protected AbstractAuditor( OriginSettingClient client, @@ -57,7 +60,8 @@ protected AbstractAuditor( String nodeName, AbstractAuditMessageFactory messageFactory, ClusterService clusterService, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + ExecutorService executorService ) { this.client = Objects.requireNonNull(client); this.auditIndexWriteAlias = Objects.requireNonNull(auditIndexWriteAlias); @@ -68,6 +72,7 @@ protected AbstractAuditor( this.backlog = new ConcurrentLinkedQueue<>(); this.indexAndAliasCreated = new AtomicBoolean(); this.indexAndAliasCreationInProgress = new AtomicBoolean(); + this.executorService = executorService; } public void audit(Level level, String resourceId, String message) { @@ -147,7 +152,16 @@ protected void indexDoc(ToXContent toXContent) { } private void writeDoc(ToXContent toXContent) { - client.index(indexRequest(toXContent), ActionListener.wrap(AbstractAuditor::onIndexResponse, AbstractAuditor::onIndexFailure)); + client.index(indexRequest(toXContent), ActionListener.wrap(AbstractAuditor::onIndexResponse, e -> { + if (e instanceof IndexNotFoundException) { + executorService.execute(() -> { + reset(); + indexDoc(toXContent); + }); + } else { + onIndexFailure(e); + } + })); } private IndexRequest indexRequest(ToXContent toXContent) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index 84091a40fb23f..b5967e6b1860c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -29,12 +29,14 @@ public class DeprecatedIndexPredicate { * @param metadata the cluster metadata * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned * @return a predicate that returns true for indices that need to be reindexed */ - public static Predicate getReindexRequiredPredicate(Metadata metadata, boolean filterToBlockedStatus) { + public static Predicate getReindexRequiredPredicate(Metadata metadata, boolean filterToBlockedStatus, boolean includeSystem) { return index -> { IndexMetadata indexMetadata = metadata.index(index); - return reindexRequired(indexMetadata, filterToBlockedStatus); + return reindexRequired(indexMetadata, filterToBlockedStatus, includeSystem); }; } @@ -45,14 +47,21 @@ public static Predicate getReindexRequiredPredicate(Metadata metadata, bo * @param indexMetadata the index metadata * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned * @return a predicate that returns true for indices that need to be reindexed */ - public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus) { + public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) { return creationVersionBeforeMinimumWritableVersion(indexMetadata) + && (includeSystem || isNotSystem(indexMetadata)) && isNotSearchableSnapshot(indexMetadata) && matchBlockedStatus(indexMetadata, filterToBlockedStatus); } + private static boolean isNotSystem(IndexMetadata indexMetadata) { + return indexMetadata.isSystem() == false; + } + private static boolean isNotSearchableSnapshot(IndexMetadata indexMetadata) { return indexMetadata.isSearchableSnapshot() == false; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RetryActionRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RetryActionRequest.java new file mode 100644 index 0000000000000..59ba94b795d85 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RetryActionRequest.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ilm.action; + +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +public class RetryActionRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable { + private String[] indices; + private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen(); + private boolean requireError = true; + + public RetryActionRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... indices) { + super(masterNodeTimeout, ackTimeout); + this.indices = indices; + } + + public RetryActionRequest(StreamInput in) throws IOException { + super(in); + this.indices = in.readStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR_8_18)) { + this.requireError = in.readBoolean(); + } + } + + @Override + public RetryActionRequest indices(String... indices) { + this.indices = indices; + return this; + } + + @Override + public String[] indices() { + return indices; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public RetryActionRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + public void requireError(boolean requireError) { + this.requireError = requireError; + } + + public boolean requireError() { + return requireError; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(indices); + indicesOptions.writeIndicesOptions(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR_8_18)) { + out.writeBoolean(requireError); + } + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(indices), indicesOptions, requireError); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + RetryActionRequest other = (RetryActionRequest) obj; + return Objects.deepEquals(indices, other.indices) + && Objects.equals(indicesOptions, other.indicesOptions) + && requireError == other.requireError; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java index f88909ba4208e..f2b2c563d7519 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java @@ -47,7 +47,7 @@ public class InferenceAction extends ActionType { public static final InferenceAction INSTANCE = new InferenceAction(); - public static final String NAME = "cluster:monitor/xpack/inference"; + public static final String NAME = "cluster:internal/xpack/inference"; public InferenceAction() { super(NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceActionProxy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceActionProxy.java new file mode 100644 index 0000000000000..512c63e554ffc --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceActionProxy.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.Objects; + +/** + * This action is used when making a REST request to the inference API. The transport handler + * will then look at the task type in the params (or retrieve it from the persisted model if it wasn't + * included in the params) to determine where this request should be routed. If the task type is chat completion + * then it will be routed to the unified chat completion handler by creating the {@link UnifiedCompletionAction}. + * If not, it will be passed along to {@link InferenceAction}. + */ +public class InferenceActionProxy extends ActionType { + public static final InferenceActionProxy INSTANCE = new InferenceActionProxy(); + public static final String NAME = "cluster:monitor/xpack/inference/post"; + + public InferenceActionProxy() { + super(NAME); + } + + public static class Request extends ActionRequest { + + private final TaskType taskType; + private final String inferenceEntityId; + private final BytesReference content; + private final XContentType contentType; + private final TimeValue timeout; + private final boolean stream; + + public Request( + TaskType taskType, + String inferenceEntityId, + BytesReference content, + XContentType contentType, + TimeValue timeout, + boolean stream + ) { + this.taskType = taskType; + this.inferenceEntityId = inferenceEntityId; + this.content = content; + this.contentType = contentType; + this.timeout = timeout; + this.stream = stream; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.taskType = TaskType.fromStream(in); + this.inferenceEntityId = in.readString(); + this.content = in.readBytesReference(); + this.contentType = in.readEnum(XContentType.class); + this.timeout = in.readTimeValue(); + + // streaming is not supported yet for transport traffic + this.stream = false; + } + + public TaskType getTaskType() { + return taskType; + } + + public String getInferenceEntityId() { + return inferenceEntityId; + } + + public BytesReference getContent() { + return content; + } + + public XContentType getContentType() { + return contentType; + } + + public TimeValue getTimeout() { + return timeout; + } + + public boolean isStreaming() { + return stream; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + taskType.writeTo(out); + out.writeString(inferenceEntityId); + out.writeBytesReference(content); + XContentHelper.writeTo(out, contentType); + out.writeTimeValue(timeout); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskType == request.taskType + && Objects.equals(inferenceEntityId, request.inferenceEntityId) + && Objects.equals(content, request.content) + && contentType == request.contentType + && timeout == request.timeout + && stream == request.stream; + } + + @Override + public int hashCode() { + return Objects.hash(taskType, inferenceEntityId, content, contentType, timeout, stream); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java index f5c852a0450ae..43c84ad914c2a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java @@ -21,7 +21,7 @@ public class UnifiedCompletionAction extends ActionType { public static final UnifiedCompletionAction INSTANCE = new UnifiedCompletionAction(); - public static final String NAME = "cluster:monitor/xpack/inference/unified"; + public static final String NAME = "cluster:internal/xpack/inference/unified"; public UnifiedCompletionAction() { super(NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java index 47bed479be44a..f1d53826d0fac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java @@ -70,8 +70,8 @@ public Request(String inferenceEntityId, BytesReference content, XContentType co public Request(StreamInput in) throws IOException { super(in); this.inferenceEntityId = in.readString(); - this.content = in.readBytesReference(); this.taskType = TaskType.fromStream(in); + this.content = in.readBytesReference(); this.contentType = in.readEnum(XContentType.class); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/UnifiedChatCompletionException.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/UnifiedChatCompletionException.java new file mode 100644 index 0000000000000..92ea327556c55 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/UnifiedChatCompletionException.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContent; + +import java.util.Iterator; +import java.util.Locale; +import java.util.Objects; + +import static java.util.Collections.emptyIterator; +import static org.elasticsearch.ExceptionsHelper.maybeError; +import static org.elasticsearch.common.collect.Iterators.concat; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; + +public class UnifiedChatCompletionException extends XContentFormattedException { + + private static final Logger log = LogManager.getLogger(UnifiedChatCompletionException.class); + private final String message; + private final String type; + @Nullable + private final String code; + @Nullable + private final String param; + + public UnifiedChatCompletionException(RestStatus status, String message, String type, @Nullable String code) { + this(status, message, type, code, null); + } + + public UnifiedChatCompletionException(RestStatus status, String message, String type, @Nullable String code, @Nullable String param) { + super(message, status); + this.message = Objects.requireNonNull(message); + this.type = Objects.requireNonNull(type); + this.code = code; + this.param = param; + } + + public UnifiedChatCompletionException( + Throwable cause, + RestStatus status, + String message, + String type, + @Nullable String code, + @Nullable String param + ) { + super(message, cause, status); + this.message = Objects.requireNonNull(message); + this.type = Objects.requireNonNull(type); + this.code = code; + this.param = param; + } + + @Override + public Iterator toXContentChunked(Params params) { + return concat( + startObject(), + startObject("error"), + optionalField("code", code), + field("message", message), + optionalField("param", param), + field("type", type), + endObject(), + endObject() + ); + } + + private static Iterator field(String key, String value) { + return Iterators.single((b, p) -> b.field(key, value)); + } + + private static Iterator optionalField(String key, String value) { + return value != null ? Iterators.single((b, p) -> b.field(key, value)) : emptyIterator(); + } + + public static UnifiedChatCompletionException fromThrowable(Throwable t) { + if (ExceptionsHelper.unwrapCause(t) instanceof UnifiedChatCompletionException e) { + return e; + } else { + return maybeError(t).map(error -> { + // we should never be throwing Error, but just in case we are, rethrow it on another thread so the JVM can handle it and + // return a vague error to the user so that they at least see something went wrong but don't leak JVM details to users + ExceptionsHelper.maybeDieOnAnotherThread(error); + var e = new RuntimeException("Fatal error while streaming response. Please retry the request."); + log.error(e.getMessage(), t); + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + e.getMessage(), + getExceptionName(e), + RestStatus.INTERNAL_SERVER_ERROR.name().toLowerCase(Locale.ROOT) + ); + }).orElseGet(() -> { + log.atDebug().withThrowable(t).log("UnifiedChatCompletionException stack trace for debugging purposes."); + var status = ExceptionsHelper.status(t); + return new UnifiedChatCompletionException( + t, + status, + t.getMessage(), + getExceptionName(t), + status.name().toLowerCase(Locale.ROOT), + null + ); + }); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/XContentFormattedException.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/XContentFormattedException.java new file mode 100644 index 0000000000000..76ff2b1d8037c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/XContentFormattedException.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.util.Iterator; +import java.util.Objects; + +/** + * Similar to {@link org.elasticsearch.ElasticsearchWrapperException}, this will wrap an Exception to generate an xContent using + * {@link ElasticsearchException#generateFailureXContent(XContentBuilder, Params, Exception, boolean)}. + * Extends {@link ElasticsearchException} to provide REST handlers the {@link #status()} method in order to set the response header. + */ +public class XContentFormattedException extends ElasticsearchException implements ChunkedToXContent { + + public static final String X_CONTENT_PARAM = "detailedErrorsEnabled"; + private final RestStatus status; + private final Throwable cause; + + public XContentFormattedException(String message, RestStatus status) { + super(message); + this.status = Objects.requireNonNull(status); + this.cause = null; + } + + public XContentFormattedException(Throwable cause, RestStatus status) { + super(cause); + this.status = Objects.requireNonNull(status); + this.cause = cause; + } + + public XContentFormattedException(String message, Throwable cause, RestStatus status) { + super(message, cause); + this.status = Objects.requireNonNull(status); + this.cause = cause; + } + + @Override + public RestStatus status() { + return status; + } + + @Override + public Iterator toXContentChunked(Params params) { + return Iterators.concat( + ChunkedToXContentHelper.startObject(), + Iterators.single( + (b, p) -> ElasticsearchException.generateFailureXContent( + b, + p, + cause instanceof Exception e ? e : this, + params.paramAsBoolean(X_CONTENT_PARAM, false) + ) + ), + Iterators.single((b, p) -> b.field("status", status.getStatus())), + ChunkedToXContentHelper.endObject() + ); + } + + @Override + public boolean isFragment() { + return super.isFragment(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index f7d2dcaf2d4a1..3da2172711acb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -407,7 +407,9 @@ public static boolean hasIndexTemplate(ClusterState state, String templateName, } public static boolean has6DigitSuffix(String indexName) { - return HAS_SIX_DIGIT_SUFFIX.test(indexName); + String[] indexParts = indexName.split("-"); + String suffix = indexParts[indexParts.length - 1]; + return HAS_SIX_DIGIT_SUFFIX.test(suffix); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java index 013d7cc21a54a..4126c1cf3cd95 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java @@ -6,6 +6,9 @@ */ package org.elasticsearch.xpack.core.security.authz.privilege; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.search.TransportSearchShardsAction; import org.elasticsearch.index.seqno.RetentionLeaseActions; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; @@ -38,12 +41,15 @@ public final class SystemPrivilege extends Privilege { RetentionLeaseActions.ADD.name() + "*", // needed for CCR to add retention leases RetentionLeaseActions.REMOVE.name() + "*", // needed for CCR to remove retention leases RetentionLeaseActions.RENEW.name() + "*", // needed for CCR to renew retention leases - "indices:admin/settings/update", // needed for DiskThresholdMonitor.markIndicesReadOnly + "indices:admin/settings/update", // needed for: DiskThresholdMonitor.markIndicesReadOnly, SystemIndexMigrator CompletionPersistentTaskAction.NAME, // needed for ShardFollowTaskCleaner "indices:data/write/*", // needed for SystemIndexMigrator "indices:data/read/*", // needed for SystemIndexMigrator "indices:admin/refresh", // needed for SystemIndexMigrator "indices:admin/aliases", // needed for SystemIndexMigrator + TransportCreateIndexAction.TYPE.name() + "*", // needed for SystemIndexMigrator + TransportAddIndexBlockAction.TYPE.name() + "*", // needed for SystemIndexMigrator + TransportUpdateSettingsAction.TYPE.name() + "*", // needed for SystemIndexMigrator TransportSearchShardsAction.TYPE.name(), // added so this API can be called with the system user by other APIs ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION.name() // needed for Security plugin reload of remote cluster credentials ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 0ab1292c228ed..85acbabf32661 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -119,6 +119,11 @@ static RoleDescriptor kibanaSystem(String name) { .indices(".ml-annotations*", ".ml-notifications*") .privileges("read", "write") .build(), + // And the reindexed indices from v7 + RoleDescriptor.IndicesPrivileges.builder() + .indices(".reindexed-v8-ml-annotations*", ".reindexed-v8-ml-notifications*") + .privileges("read", "write") + .build(), // APM agent configuration - system index defined in KibanaPlugin RoleDescriptor.IndicesPrivileges.builder() @@ -439,8 +444,10 @@ static RoleDescriptor kibanaSystem(String name) { .indices( "logs-wiz.vulnerability-*", "logs-wiz.cloud_configuration_finding-*", + "logs-wiz.cloud_configuration_finding_full_posture-*", "logs-google_scc.finding-*", "logs-aws.securityhub_findings-*", + "logs-aws.securityhub_findings_full_posture-*", "logs-aws.inspector-*", "logs-amazon_security_lake.findings-*", "logs-qualys_vmdr.asset_host_detection-*", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java index 1229d62dce047..e2bf98749082f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.rollover.LazyRolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.bulk.TransportBulkAction; @@ -29,6 +30,8 @@ import org.elasticsearch.action.search.TransportSearchScrollAction; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.support.MetadataUtils; @@ -207,24 +210,29 @@ public class InternalUsers { TransportDeleteIndexAction.TYPE.name(), "indices:admin/data_stream/index/reindex", "indices:admin/index/create_from_source", + "indices:admin/index/copy_lifecycle_index_metadata", TransportAddIndexBlockAction.TYPE.name(), OpenIndexAction.NAME, TransportCloseIndexAction.NAME, TransportCreateIndexAction.TYPE.name(), TransportClusterSearchShardsAction.TYPE.name(), + GetSettingsAction.NAME, TransportUpdateSettingsAction.TYPE.name(), RefreshAction.NAME, ReindexAction.NAME, + FreezeIndexAction.NAME, TransportSearchAction.NAME, TransportBulkAction.NAME, TransportIndexAction.NAME, TransportSearchScrollAction.TYPE.name(), - ModifyDataStreamsAction.NAME + ModifyDataStreamsAction.NAME, + ILMActions.RETRY.name() ) - .allowRestrictedIndices(false) + .allowRestrictedIndices(true) .build() }, null, null, + new String[] {}, MetadataUtils.DEFAULT_RESERVED_METADATA, Map.of() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java index 6424136eb1a7c..9a35b8f13d4c1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java @@ -146,7 +146,7 @@ public static SslKeyConfig createKeyConfig( boolean acceptNonSecurePasswords ) { final SslSettingsLoader settingsLoader = new SslSettingsLoader(settings, prefix, acceptNonSecurePasswords); - return settingsLoader.buildKeyConfig(environment.configFile()); + return settingsLoader.buildKeyConfig(environment.configDir()); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java index cb32ef2d8b187..e75fe0ab26f35 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.security.AccessControlException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -109,7 +108,7 @@ private static void startWatching( fileWatcher.addListener(changeListener); try { resourceWatcherService.add(fileWatcher, Frequency.HIGH); - } catch (IOException | AccessControlException e) { + } catch (IOException | SecurityException e) { logger.error("failed to start watching directory [{}] for ssl configurations [{}] - {}", path, configurations, e); } }); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java index cb55de79342b8..f9b27daa8f8c1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java @@ -128,7 +128,7 @@ protected SslTrustConfig buildTrustConfig( } public SslConfiguration load(Environment env) { - return load(env.configFile()); + return load(env.configDir()); } public static SslConfiguration load(Settings settings, String prefix, Environment env) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 483ee72ef3fce..1144d1a87433e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; @@ -840,9 +841,8 @@ public void onFailure(Exception e) { void onRolloversBulkResponse(Collection rolloverResponses) { for (RolloverResponse rolloverResponse : rolloverResponses) { - if (rolloverResponse.isRolledOver() == false) { - logger.warn("rollover of the [{}] index [{}] failed", getOrigin(), rolloverResponse.getOldIndex()); - } + assert rolloverResponse.isLazy() && rolloverResponse.isRolledOver() == false + : Strings.format("Expected rollover of the [%s] index [%s] to be lazy", getOrigin(), rolloverResponse.getOldIndex()); } } diff --git a/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml index 4e0266b06bbb0..f543e39972a78 100644 --- a/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,8 +1,23 @@ +org.elasticsearch.xcore: + - files: + - relative_path: "" + relative_to: config + mode: read +org.elasticsearch.sslconfig: + - files: + - relative_path: "" + relative_to: config + mode: read org.apache.httpcomponents.httpclient: - outbound_network # For SamlRealm + - manage_threads org.apache.httpcomponents.httpcore.nio: - outbound_network + - manage_threads +org.apache.httpcomponents.httpasyncclient: + - manage_threads unboundid.ldapsdk: + - manage_threads - write_system_properties: properties: - java.security.auth.login.config diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java index ab6e7356a6e02..f7432a59040da 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java @@ -143,7 +143,7 @@ public List loadExtensions(Class extensionPointType) { Environment mockEnvironment = mock(Environment.class); when(mockEnvironment.settings()).thenReturn(Settings.builder().build()); - when(mockEnvironment.configFile()).thenReturn(PathUtils.get("")); + when(mockEnvironment.configDir()).thenReturn(PathUtils.get("")); // ensure createComponents does not influence the results Plugin.PluginServices services = mock(Plugin.PluginServices.class); when(services.clusterService()).thenReturn(mock(ClusterService.class)); @@ -187,7 +187,7 @@ public List loadExtensions(Class extensionPointType) { }); Environment mockEnvironment = mock(Environment.class); when(mockEnvironment.settings()).thenReturn(Settings.builder().build()); - when(mockEnvironment.configFile()).thenReturn(PathUtils.get("")); + when(mockEnvironment.configDir()).thenReturn(PathUtils.get("")); Plugin.PluginServices services = mock(Plugin.PluginServices.class); when(services.clusterService()).thenReturn(mock(ClusterService.class)); when(services.threadPool()).thenReturn(mock(ThreadPool.class)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java index 92feb4de81aa3..3c81c52277b0e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java @@ -30,8 +30,22 @@ public class XPackSettingsTests extends ESTestCase { public void testDefaultSSLCiphers() { - assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_RSA_WITH_AES_128_CBC_SHA")); - assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_RSA_WITH_AES_256_CBC_SHA")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_AES_256_GCM_SHA384")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_AES_128_GCM_SHA256")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384")); + + if (Runtime.version().feature() < 24) { + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_RSA_WITH_AES_256_CBC_SHA256")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_RSA_WITH_AES_128_CBC_SHA256")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_RSA_WITH_AES_256_CBC_SHA")); + assertThat(XPackSettings.DEFAULT_CIPHERS, hasItem("TLS_RSA_WITH_AES_128_CBC_SHA")); + } else { + assertThat(XPackSettings.DEFAULT_CIPHERS, not(hasItem("TLS_RSA_WITH_AES_256_CBC_SHA256"))); + assertThat(XPackSettings.DEFAULT_CIPHERS, not(hasItem("TLS_RSA_WITH_AES_128_CBC_SHA256"))); + assertThat(XPackSettings.DEFAULT_CIPHERS, not(hasItem("TLS_RSA_WITH_AES_256_CBC_SHA"))); + assertThat(XPackSettings.DEFAULT_CIPHERS, not(hasItem("TLS_RSA_WITH_AES_128_CBC_SHA"))); + } } public void testChaCha20InCiphersOnJdk12Plus() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java index bcf777906bb7c..13d15a4ab0b07 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditorTests.java @@ -34,8 +34,10 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; @@ -233,6 +235,40 @@ public void testAuditingBeforeTemplateInstalled() throws Exception { verify(client, times(1)).execute(eq(TransportIndexAction.TYPE), any(), any()); } + public void testRecreateTemplateWhenDeleted() throws Exception { + CountDownLatch writeSomeDocsBeforeTemplateLatch = new CountDownLatch(1); + AbstractAuditor auditor = createTestAuditorWithoutTemplate( + writeSomeDocsBeforeTemplateLatch + ); + + auditor.info("foobar", "Here is my info to queue"); + + verify(client, never()).execute(eq(TransportIndexAction.TYPE), any(), any()); + // fire the put template response + writeSomeDocsBeforeTemplateLatch.countDown(); + + assertBusy(() -> verify(client, times(1)).execute(eq(TransportPutComposableIndexTemplateAction.TYPE), any(), any())); + assertBusy(() -> verify(client, times(1)).execute(eq(TransportCreateIndexAction.TYPE), any(), any())); + + // the back log will be written some point later + assertBusy(() -> verify(client, times(1)).execute(eq(TransportBulkAction.TYPE), any(), any())); + + // "delete" the index + doAnswer(ans -> { + ActionListener listener = ans.getArgument(2); + listener.onFailure(new IndexNotFoundException("some index")); + return null; + }).when(client).execute(eq(TransportIndexAction.TYPE), any(), any()); + + // audit more data + auditor.info("foobar", "Here is another message"); + + // verify the template is recreated and the audit message is processed + assertBusy(() -> verify(client, times(2)).execute(eq(TransportPutComposableIndexTemplateAction.TYPE), any(), any())); + assertBusy(() -> verify(client, times(2)).execute(eq(TransportCreateIndexAction.TYPE), any(), any())); + assertBusy(() -> verify(client, times(2)).execute(eq(TransportBulkAction.TYPE), any(), any())); + } + public void testMaxBufferSize() throws Exception { CountDownLatch writeSomeDocsBeforeTemplateLatch = new CountDownLatch(1); AbstractAuditor auditor = createTestAuditorWithoutTemplate( @@ -358,7 +394,8 @@ public static class TestAuditor extends AbstractAuditor new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -141,7 +141,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -214,7 +214,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -287,7 +287,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -360,7 +360,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -434,7 +434,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -508,7 +508,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), @@ -582,7 +582,7 @@ protected Response mutateInstanceForVersion(Response instance, TransportVersion pipelineStat -> new IngestStats.PipelineStat( pipelineStat.pipelineId(), pipelineStat.stats(), - new IngestStats.ByteStats(0, 0) + IngestStats.ByteStats.IDENTITY ) ) .toList(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index 909b668c1731b..4e7c5f4f6339a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -382,6 +382,13 @@ public void testIndexIsReadWriteCompatibleInV9() { assertFalse(MlIndexAndAlias.indexIsReadWriteCompatibleInV9(IndexVersions.V_7_17_0)); } + public void testHas6DigitSuffix() { + assertTrue(MlIndexAndAlias.has6DigitSuffix("index-000001")); + assertFalse(MlIndexAndAlias.has6DigitSuffix("index1")); + assertFalse(MlIndexAndAlias.has6DigitSuffix("index-foo")); + assertFalse(MlIndexAndAlias.has6DigitSuffix("index000001")); + } + private void createIndexAndAliasIfNecessary(ClusterState clusterState) { MlIndexAndAlias.createIndexAndAliasIfNecessary( client, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 19a077448fba7..e57150da5957f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -1622,8 +1622,10 @@ public void testKibanaSystemRole() { Arrays.asList( "logs-wiz.vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13)), "logs-wiz.cloud_configuration_finding-" + randomAlphaOfLength(randomIntBetween(0, 13)), + "logs-wiz.cloud_configuration_finding_full_posture-" + randomAlphaOfLength(randomIntBetween(0, 13)), "logs-google_scc.finding-" + randomAlphaOfLength(randomIntBetween(0, 13)), "logs-aws.securityhub_findings-" + randomAlphaOfLength(randomIntBetween(0, 13)), + "logs-aws.securityhub_findings_full_posture-" + randomAlphaOfLength(randomIntBetween(0, 13)), "logs-aws.inspector-" + randomAlphaOfLength(randomIntBetween(0, 13)), "logs-amazon_security_lake.findings-" + randomAlphaOfLength(randomIntBetween(0, 13)), "logs-qualys_vmdr.asset_host_detection-" + randomAlphaOfLength(randomIntBetween(0, 13)), @@ -4219,7 +4221,7 @@ public void testInferenceAdminRole() { assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES); - assertTrue(role.cluster().check("cluster:monitor/xpack/inference", request, authentication)); + assertTrue(role.cluster().check("cluster:monitor/xpack/inference/post", request, authentication)); assertTrue(role.cluster().check("cluster:monitor/xpack/inference/get", request, authentication)); assertTrue(role.cluster().check("cluster:admin/xpack/inference/put", request, authentication)); assertTrue(role.cluster().check("cluster:admin/xpack/inference/delete", request, authentication)); @@ -4239,10 +4241,9 @@ public void testInferenceUserRole() { assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES); - assertTrue(role.cluster().check("cluster:monitor/xpack/inference", request, authentication)); + assertTrue(role.cluster().check("cluster:monitor/xpack/inference/post", request, authentication)); assertTrue(role.cluster().check("cluster:monitor/xpack/inference/get", request, authentication)); assertFalse(role.cluster().check("cluster:admin/xpack/inference/put", request, authentication)); - assertTrue(role.cluster().check("cluster:monitor/xpack/inference/unified", request, authentication)); assertFalse(role.cluster().check("cluster:admin/xpack/inference/delete", request, authentication)); assertTrue(role.cluster().check("cluster:monitor/xpack/ml/trained_models/deployment/infer", request, authentication)); assertFalse(role.cluster().check("cluster:admin/xpack/ml/trained_models/deployment/start", request, authentication)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index 9e26444040b03..a024957375372 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -116,6 +116,7 @@ public class TestRestrictedIndices { .build(), Map.of(), List.of("fleet", "kibana"), + "fleet", null ) ) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java index a800266a33a18..de214327ae841 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java @@ -35,7 +35,7 @@ public void testParseCipherSettingsWithoutPrefix() { final Settings settings = Settings.builder() .put("cipher_suites.0", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256") .put("cipher_suites.1", "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256") - .put("cipher_suites.2", "TLS_RSA_WITH_AES_128_CBC_SHA256") + .put("cipher_suites.2", Runtime.version().feature() < 24 ? "TLS_RSA_WITH_AES_128_CBC_SHA256" : "TLS_AES_256_GCM_SHA384") .build(); assertThat( ssl.ciphers.get(settings), @@ -43,7 +43,7 @@ public void testParseCipherSettingsWithoutPrefix() { Arrays.asList( "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", - "TLS_RSA_WITH_AES_128_CBC_SHA256" + Runtime.version().feature() < 24 ? "TLS_RSA_WITH_AES_128_CBC_SHA256" : "TLS_AES_256_GCM_SHA384" ) ) ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java index d03595c39877b..79ff817061a01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java @@ -229,7 +229,7 @@ public void testKeystorePassword() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, PASSWORD, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, PASSWORD, KEY_MGR_ALGORITHM, environment.configDir())) ); } @@ -244,7 +244,7 @@ public void testKeystorePasswordBackcompat() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, PASSWORD, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, PASSWORD, KEY_MGR_ALGORITHM, environment.configDir())) ); assertSettingDeprecationsAndWarnings(new Setting[] { configurationSettings.x509KeyPair.legacyKeystorePassword }); } @@ -263,7 +263,7 @@ public void testKeystoreKeyPassword() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configDir())) ); } @@ -279,7 +279,7 @@ public void testKeystoreKeyPasswordBackcompat() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig("path", PASSWORD, "type", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configDir())) ); assertSettingDeprecationsAndWarnings( new Setting[] { @@ -298,7 +298,7 @@ public void testInferKeystoreTypeFromJksFile() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig("xpack/tls/path.jks", PASSWORD, "jks", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig("xpack/tls/path.jks", PASSWORD, "jks", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configDir())) ); } @@ -314,7 +314,7 @@ public void testInferKeystoreTypeFromPkcs12File() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig(path, PASSWORD, "PKCS12", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig(path, PASSWORD, "PKCS12", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configDir())) ); } @@ -328,7 +328,7 @@ public void testInferKeystoreTypeFromUnrecognised() { StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); assertThat( ksKeyInfo, - equalTo(new StoreKeyConfig("xpack/tls/path.foo", PASSWORD, "jks", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configFile())) + equalTo(new StoreKeyConfig("xpack/tls/path.foo", PASSWORD, "jks", null, KEYPASS, KEY_MGR_ALGORITHM, environment.configDir())) ); } @@ -347,10 +347,7 @@ public void testExplicitKeystoreType() { SslConfiguration sslConfiguration = getSslConfiguration(settings); assertThat(sslConfiguration.keyConfig(), instanceOf(StoreKeyConfig.class)); StoreKeyConfig ksKeyInfo = (StoreKeyConfig) sslConfiguration.keyConfig(); - assertThat( - ksKeyInfo, - equalTo(new StoreKeyConfig(path, PASSWORD, type, null, KEYPASS, KEY_MGR_ALGORITHM, environment.configFile())) - ); + assertThat(ksKeyInfo, equalTo(new StoreKeyConfig(path, PASSWORD, type, null, KEYPASS, KEY_MGR_ALGORITHM, environment.configDir()))); } public void testThatEmptySettingsAreEqual() { diff --git a/x-pack/plugin/core/src/yamlRestTest/java/org/elasticsearch/license/XPackCoreClientYamlTestSuiteIT.java b/x-pack/plugin/core/src/yamlRestTest/java/org/elasticsearch/license/XPackCoreClientYamlTestSuiteIT.java index 105df6dbeca4a..a6c658bb1fe60 100644 --- a/x-pack/plugin/core/src/yamlRestTest/java/org/elasticsearch/license/XPackCoreClientYamlTestSuiteIT.java +++ b/x-pack/plugin/core/src/yamlRestTest/java/org/elasticsearch/license/XPackCoreClientYamlTestSuiteIT.java @@ -13,11 +13,24 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class XPackCoreClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password") + .systemProperty("es.queryable_built_in_roles_enabled", "false") + .build(); + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password")); public XPackCoreClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { @@ -33,4 +46,9 @@ public static Iterable parameters() throws Exception { protected Settings restClientSettings() { return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 8b1c13f3152e8..ac2f88746eef2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -102,6 +102,9 @@ "unprivileged": { "type": "boolean" }, + "fips": { + "type": "boolean" + }, "version": { "type": "text", "fields": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json index 7457dce805eca..fcb299115ffd2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json @@ -786,6 +786,45 @@ }, "sampling": { "properties": { + "tail": { + "properties": { + "dynamic_service_groups": { + "type": "long" + }, + "events": { + "properties": { + "dropped": { + "type": "long" + }, + "failed_writes": { + "type": "long" + }, + "head_unsampled": { + "type": "long" + }, + "processed": { + "type": "long" + }, + "sampled": { + "type": "long" + }, + "stored": { + "type": "long" + } + } + }, + "storage": { + "properties": { + "lsm_size": { + "type": "long" + }, + "value_log_size": { + "type": "long" + } + } + } + } + }, "transactions_dropped": { "type": "long" } @@ -2219,6 +2258,54 @@ }, "sampling": { "properties": { + "tail": { + "properties": { + "dynamic_service_groups": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.dynamic_service_groups" + }, + "events": { + "properties": { + "dropped": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.events.dropped" + }, + "failed_writes": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.events.failed_writes" + }, + "head_unsampled": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.events.head_unsampled" + }, + "processed": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.events.processed" + }, + "sampled": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.events.sampled" + }, + "stored": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.events.stored" + } + } + }, + "storage": { + "properties": { + "lsm_size": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.storage.lsm_size" + }, + "value_log_size": { + "type": "alias", + "path": "beat.stats.apm_server.sampling.tail.storage.value_log_size" + } + } + } + } + }, "transactions_dropped": { "type": "alias", "path": "beat.stats.apm_server.sampling.transactions_dropped" diff --git a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json index d699317c29da3..e1a8b5cc37060 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json @@ -966,6 +966,45 @@ }, "sampling": { "properties": { + "tail": { + "properties": { + "dynamic_service_groups": { + "type": "long" + }, + "events": { + "properties": { + "dropped": { + "type": "long" + }, + "failed_writes": { + "type": "long" + }, + "head_unsampled": { + "type": "long" + }, + "processed": { + "type": "long" + }, + "sampled": { + "type": "long" + }, + "stored": { + "type": "long" + } + } + }, + "storage": { + "properties": { + "lsm_size": { + "type": "long" + }, + "value_log_size": { + "type": "long" + } + } + } + } + }, "transactions_dropped": { "type": "long" } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/reindex-data-stream-pipeline.json b/x-pack/plugin/core/template-resources/src/main/resources/reindex-data-stream-pipeline.json new file mode 100644 index 0000000000000..e8c3352131700 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/reindex-data-stream-pipeline.json @@ -0,0 +1,16 @@ +{ + "description": "This pipeline sanitizes documents that are being reindexed into a data stream using the reindex data stream API. It is an internal pipeline and should not be modified.", + "processors": [ + { + "set": { + "field": "@timestamp", + "value": 0, + "override": false + } + } + ], + "_meta": { + "managed": true + }, + "version": ${xpack.migrate.reindex.pipeline.version} +} diff --git a/x-pack/plugin/deprecation/src/main/java/module-info.java b/x-pack/plugin/deprecation/src/main/java/module-info.java index f9a86839ad6f2..4c46205df4f0c 100644 --- a/x-pack/plugin/deprecation/src/main/java/module-info.java +++ b/x-pack/plugin/deprecation/src/main/java/module-info.java @@ -13,6 +13,7 @@ requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; requires log4j2.ecs.layout; + requires org.apache.lucene.core; exports org.elasticsearch.xpack.deprecation to org.elasticsearch.server; exports org.elasticsearch.xpack.deprecation.logging to org.elasticsearch.server; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java new file mode 100644 index 0000000000000..cc21f0b2cd711 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecker.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.TriConsumer; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Cluster-specific deprecation checks, this is used to populate the {@code cluster_settings} field + */ +public class ClusterDeprecationChecker { + + private static final Logger logger = LogManager.getLogger(ClusterDeprecationChecker.class); + private final List, List>> CHECKS = List.of( + this::checkTransformSettings + ); + private final NamedXContentRegistry xContentRegistry; + + ClusterDeprecationChecker(NamedXContentRegistry xContentRegistry) { + this.xContentRegistry = xContentRegistry; + } + + public List check(ClusterState clusterState, List transformConfigs) { + List allIssues = new ArrayList<>(); + CHECKS.forEach(check -> check.apply(clusterState, transformConfigs, allIssues)); + return allIssues; + } + + private void checkTransformSettings( + ClusterState clusterState, + List transformConfigs, + List allIssues + ) { + for (var config : transformConfigs) { + try { + allIssues.addAll(config.checkForDeprecations(xContentRegistry)); + } catch (IOException e) { + logger.warn("failed to check transformation settings for '" + config.getId() + "'", e); + } + } + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index ccd2adec1468c..973434d19a398 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -18,13 +18,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; import static java.util.Map.entry; import static java.util.Map.ofEntries; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; /** * Checks the data streams for deprecation warnings. @@ -44,10 +44,24 @@ public DataStreamDeprecationChecker(IndexNameExpressionResolver indexNameExpress /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + public Map> check(ClusterState clusterState) { List dataStreamNames = indexNameExpressionResolver.dataStreamNames( clusterState, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN @@ -58,9 +72,14 @@ public Map> check(ClusterState clusterState, Depr Map> dataStreamIssues = new HashMap<>(); for (String dataStreamName : dataStreamNames) { DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName); - List issuesForSingleDataStream = filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState)); - if (issuesForSingleDataStream.isEmpty() == false) { - dataStreamIssues.put(dataStreamName, issuesForSingleDataStream); + if (dataStream.isSystem() == false) { + List issuesForSingleDataStream = DATA_STREAM_CHECKS.stream() + .map(c -> c.apply(dataStream, clusterState)) + .filter(Objects::nonNull) + .toList(); + if (issuesForSingleDataStream.isEmpty() == false) { + dataStreamIssues.put(dataStreamName, issuesForSingleDataStream); + } } } return dataStreamIssues.isEmpty() ? Map.of() : dataStreamIssues; @@ -102,7 +121,7 @@ static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterSta + "OK to remain read-only after upgrade", false, ofEntries( - entry("reindex_required", true), + entry("reindex_required", false), entry("total_backing_indices", backingIndices.size()), entry("ignored_indices_requiring_upgrade_count", ignoredIndices.size()), entry("ignored_indices_requiring_upgrade", ignoredIndices) @@ -118,7 +137,7 @@ private static Set getReindexRequiredIndices( boolean filterToBlockedStatus ) { return backingIndices.stream() - .filter(DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterState.metadata(), filterToBlockedStatus)) + .filter(DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterState.metadata(), filterToBlockedStatus, false)) .map(Index::getName) .collect(Collectors.toUnmodifiableSet()); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index 85b7c89e7cb85..2c8b95e378375 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -33,7 +33,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.SKIP_DEPRECATIONS_SETTING; +import static org.elasticsearch.xpack.deprecation.TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING; import static org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent.DEPRECATION_INDEXING_FLUSH_INTERVAL; /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java deleted file mode 100644 index e1d18d491423a..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Class containing all the cluster, node, and index deprecation checks that will be served - * by the {@link DeprecationInfoAction}. - */ -public class DeprecationChecks { - - public static final Setting> SKIP_DEPRECATIONS_SETTING = Setting.stringListSetting( - "deprecation.skip_deprecated_settings", - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private DeprecationChecks() {} - - static List> CLUSTER_SETTINGS_CHECKS = List.of(); - - static final List< - NodeDeprecationCheck> NODE_SETTINGS_CHECKS = List - .of( - NodeDeprecationChecks::checkMultipleDataPaths, - NodeDeprecationChecks::checkDataPathsList, - NodeDeprecationChecks::checkSharedDataPathSetting, - NodeDeprecationChecks::checkReservedPrefixedRealmNames, - NodeDeprecationChecks::checkSingleDataNodeWatermarkSetting, - NodeDeprecationChecks::checkExporterUseIngestPipelineSettings, - NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting, - NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting, - NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, - NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery, - NodeDeprecationChecks::checkMonitoringSettingCollectIndices, - NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersHost, - NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername, - NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass, - NodeDeprecationChecks::checkMonitoringSettingExportersSSL, - NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase, - NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersHeaders, - NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout, - NodeDeprecationChecks::checkMonitoringSettingExportersEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersType, - NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled, - NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist, - NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat, - NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts, - NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled, - NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled, - NodeDeprecationChecks::checkMonitoringSettingCollectionInterval, - NodeDeprecationChecks::checkScriptContextCache, - NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, - NodeDeprecationChecks::checkScriptContextCacheSizeSetting, - NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, - NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting, - NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, - NodeDeprecationChecks::checkEqlEnabledSetting, - NodeDeprecationChecks::checkNodeAttrData, - NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, - NodeDeprecationChecks::checkTracingApmSettings - ); - - /** - * helper utility function to reduce repeat of running a specific {@link List} of checks. - * - * @param checks The functional checks to execute using the mapper function - * @param mapper The function that executes the lambda check with the appropriate arguments - * @param The signature of the check (BiFunction, Function, including the appropriate arguments) - * @return The list of {@link DeprecationIssue} that were found in the cluster - */ - static List filterChecks(List checks, Function mapper) { - return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList()); - } - - @FunctionalInterface - public interface NodeDeprecationCheck { - R apply(A first, B second, C third, D fourth); - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java index c89e61fbcf24d..1fceb917ece53 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java @@ -12,41 +12,25 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -59,93 +43,6 @@ private DeprecationInfoAction() { super(NAME); } - /** - * helper utility function to reduce repeat of running a specific {@link Set} of checks. - * - * @param checks The functional checks to execute using the mapper function - * @param mapper The function that executes the lambda check with the appropriate arguments - * @param The signature of the check (BiFunction, Function, including the appropriate arguments) - * @return The list of {@link DeprecationIssue} that were found in the cluster - */ - public static List filterChecks(List checks, Function mapper) { - return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList()); - } - - /** - * This method rolls up DeprecationIssues that are identical but on different nodes. It also roles up DeprecationIssues that are - * identical (and on different nodes) except that they differ in the removable settings listed in their meta object. We roll these up - * by taking the intersection of all removable settings in otherwise identical DeprecationIssues. That way we don't claim that a - * setting can be automatically removed if any node has it in its elasticsearch.yml. - * @param response - * @return - */ - private static List mergeNodeIssues(NodesDeprecationCheckResponse response) { - // A collection whose values are lists of DeprecationIssues that differ only by meta values (if that): - Collection>> issuesToMerge = getDeprecationIssuesThatDifferOnlyByMeta(response.getNodes()); - // A map of DeprecationIssues (containing only the intersection of removable settings) to the nodes they are seen on - Map> issueToListOfNodesMap = getMergedIssuesToNodesMap(issuesToMerge); - - return issueToListOfNodesMap.entrySet().stream().map(entry -> { - DeprecationIssue issue = entry.getKey(); - String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; - return new DeprecationIssue( - issue.getLevel(), - issue.getMessage(), - issue.getUrl(), - details + "(nodes impacted: " + entry.getValue() + ")", - issue.isResolveDuringRollingUpgrade(), - issue.getMeta() - ); - }).collect(Collectors.toList()); - } - - /* - * This method pulls all the DeprecationIssues from the given nodeResponses, and buckets them into lists of DeprecationIssues that - * differ at most by meta values (if that). The returned tuples also contain the node name the deprecation issue was found on. If all - * nodes in the cluster were configured identically then all tuples in a list will differ only by the node name. - */ - private static Collection>> getDeprecationIssuesThatDifferOnlyByMeta( - List nodeResponses - ) { - Map>> issuesToMerge = new HashMap<>(); - for (NodesDeprecationCheckAction.NodeResponse resp : nodeResponses) { - for (DeprecationIssue issue : resp.getDeprecationIssues()) { - issuesToMerge.computeIfAbsent( - new DeprecationIssue( - issue.getLevel(), - issue.getMessage(), - issue.getUrl(), - issue.getDetails(), - issue.isResolveDuringRollingUpgrade(), - null // Intentionally removing meta from the key so that it's not taken into account for equality - ), - (key) -> new ArrayList<>() - ).add(new Tuple<>(issue, resp.getNode().getName())); - } - } - return issuesToMerge.values(); - } - - /* - * At this point we have one DeprecationIssue per node for a given deprecation. This method rolls them up into a single DeprecationIssue - * with a list of nodes that they appear on. If two DeprecationIssues on two different nodes differ only by the set of removable - * settings (i.e. they have different elasticsearch.yml configurations) then this method takes the intersection of those settings when - * it rolls them up. - */ - private static Map> getMergedIssuesToNodesMap( - Collection>> issuesToMerge - ) { - Map> issueToListOfNodesMap = new HashMap<>(); - for (List> similarIssues : issuesToMerge) { - DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( - similarIssues.stream().map(Tuple::v1).toList() - ); - issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) - .addAll(similarIssues.stream().map(Tuple::v2).toList()); - } - return issueToListOfNodesMap; - } - public static class Response extends ActionResponse implements ToXContentObject { static final Set RESERVED_NAMES = Set.of( "cluster_settings", @@ -289,143 +186,6 @@ public int hashCode() { return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, resourceDeprecationIssues, pluginSettingsIssues); } - /** - * This is the function that does the bulk of the logic of taking the appropriate ES dependencies - * like {@link NodeInfo}, {@link ClusterState}. Alongside these objects and the list of deprecation checks, - * this function will run through all the checks and build out the final list of issues that exist in the - * cluster. - * - * @param state The cluster state - * @param indexNameExpressionResolver Used to resolve indices into their concrete names - * @param request The originating request containing the index expressions to evaluate - * @param nodeDeprecationResponse The response containing the deprecation issues found on each node - * @param clusterSettingsChecks The list of cluster-level checks - * @param pluginSettingIssues this map gets modified to move transform deprecation issues into cluster_settings - * @param skipTheseDeprecatedSettings the settings that will be removed from cluster metadata and the index metadata of all the - * indexes specified by indexNames - * @param resourceDeprecationCheckers these are checkers that take as input the cluster state and return a map from resource type - * to issues grouped by the resource name. - * @return The list of deprecation issues found in the cluster - */ - public static DeprecationInfoAction.Response from( - ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - Request request, - NodesDeprecationCheckResponse nodeDeprecationResponse, - List> clusterSettingsChecks, - Map> pluginSettingIssues, - List skipTheseDeprecatedSettings, - List resourceDeprecationCheckers - ) { - assert Transports.assertNotTransportThread("walking mappings in indexSettingsChecks is expensive"); - // Allow system index access here to prevent deprecation warnings when we call this API - String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, request); - ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); - List clusterSettingsIssues = filterChecks( - clusterSettingsChecks, - (c) -> c.apply(stateWithSkippedSettingsRemoved) - ); - List nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse); - - Map>> resourceDeprecationIssues = new HashMap<>(); - for (ResourceDeprecationChecker resourceDeprecationChecker : resourceDeprecationCheckers) { - Map> issues = resourceDeprecationChecker.check(stateWithSkippedSettingsRemoved, request); - if (issues.isEmpty() == false) { - resourceDeprecationIssues.put(resourceDeprecationChecker.getName(), issues); - } - } - - // WORKAROUND: move transform deprecation issues into cluster_settings - List transformDeprecations = pluginSettingIssues.remove( - TransformDeprecationChecker.TRANSFORM_DEPRECATION_KEY - ); - if (transformDeprecations != null) { - clusterSettingsIssues.addAll(transformDeprecations); - } - - return new DeprecationInfoAction.Response( - clusterSettingsIssues, - nodeSettingsIssues, - resourceDeprecationIssues, - pluginSettingIssues - ); - } - } - - /** - * - * @param state The cluster state to modify - * @param indexNames The names of the indexes whose settings need to be filtered - * @param skipTheseDeprecatedSettings The settings that will be removed from cluster metadata and the index metadata of all the - * indexes specified by indexNames - * @return A modified cluster state with the given settings removed - */ - private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { - // Short-circuit, no need to reconstruct the cluster state if there are no settings to remove - if (skipTheseDeprecatedSettings == null || skipTheseDeprecatedSettings.isEmpty()) { - return state; - } - ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); - Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); - metadataBuilder.transientSettings( - metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ); - metadataBuilder.persistentSettings( - metadataBuilder.persistentSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ); - Map indicesBuilder = new HashMap<>(state.getMetadata().indices()); - for (String indexName : indexNames) { - IndexMetadata indexMetadata = state.getMetadata().index(indexName); - IndexMetadata.Builder filteredIndexMetadataBuilder = new IndexMetadata.Builder(indexMetadata); - Settings filteredSettings = indexMetadata.getSettings() - .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false); - filteredIndexMetadataBuilder.settings(filteredSettings); - indicesBuilder.put(indexName, filteredIndexMetadataBuilder.build()); - } - metadataBuilder.componentTemplates(state.metadata().componentTemplates().entrySet().stream().map(entry -> { - String templateName = entry.getKey(); - ComponentTemplate componentTemplate = entry.getValue(); - Template template = componentTemplate.template(); - if (template.settings() == null || template.settings().isEmpty()) { - return Tuple.tuple(templateName, componentTemplate); - } - return Tuple.tuple( - templateName, - new ComponentTemplate( - Template.builder(template) - .settings(template.settings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)) - .build(), - componentTemplate.version(), - componentTemplate.metadata(), - componentTemplate.deprecated() - ) - ); - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - metadataBuilder.indexTemplates(state.metadata().templatesV2().entrySet().stream().map(entry -> { - String templateName = entry.getKey(); - ComposableIndexTemplate indexTemplate = entry.getValue(); - Template template = indexTemplate.template(); - if (templateName == null || template.settings() == null || template.settings().isEmpty()) { - return Tuple.tuple(templateName, indexTemplate); - } - return Tuple.tuple( - templateName, - indexTemplate.toBuilder() - .template( - Template.builder(indexTemplate.template()) - .settings( - indexTemplate.template() - .settings() - .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) - ) - ) - .build() - ); - }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - - metadataBuilder.indices(indicesBuilder); - clusterStateBuilder.metadata(metadataBuilder); - return clusterStateBuilder.build(); } public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java index 6d7f860f645f1..f96fae6343b9f 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationChecker.java @@ -19,9 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; @@ -33,17 +33,28 @@ public class IlmPolicyDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "ilm_policies"; - private static final List> CHECKS = List.of( - IlmPolicyDeprecationChecker::checkLegacyTiers, - IlmPolicyDeprecationChecker::checkFrozenAction - ); + private final List> checks = List.of(this::checkLegacyTiers, this::checkFrozenAction); /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + Map> check(ClusterState clusterState) { IndexLifecycleMetadata lifecycleMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE); if (lifecycleMetadata == null || lifecycleMetadata.getPolicyMetadatas().isEmpty()) { return Map.of(); @@ -53,7 +64,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); LifecyclePolicyMetadata policyMetadata = entry.getValue(); - List issuesForSinglePolicy = filterChecks(CHECKS, c -> c.apply(policyMetadata.getPolicy())); + List issuesForSinglePolicy = checks.stream() + .map(c -> c.apply(policyMetadata.getPolicy())) + .filter(Objects::nonNull) + .toList(); if (issuesForSinglePolicy.isEmpty() == false) { issues.put(name, issuesForSinglePolicy); } @@ -61,7 +75,7 @@ public Map> check(ClusterState clusterState, Depr return issues.isEmpty() ? Map.of() : issues; } - static DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { + private DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { for (Phase phase : policy.getPhases().values()) { AllocateAction allocateAction = (AllocateAction) phase.getActions().get(AllocateAction.NAME); if (allocateAction != null) { @@ -82,7 +96,7 @@ static DeprecationIssue checkLegacyTiers(LifecyclePolicy policy) { return null; } - static DeprecationIssue checkFrozenAction(LifecyclePolicy policy) { + private DeprecationIssue checkFrozenAction(LifecyclePolicy policy) { for (Phase phase : policy.getPhases().values()) { if (phase.getActions().containsKey(FreezeAction.NAME)) { return new DeprecationIssue( diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index d9eb30b9909c0..dfd308d23afda 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -10,25 +10,30 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.time.LegacyFormatNames; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.engine.frozen.FrozenEngine; import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; @@ -40,20 +45,36 @@ public class IndexDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "index_settings"; private final IndexNameExpressionResolver indexNameExpressionResolver; - private final Map> indexToTransformIds; + private final List>, DeprecationIssue>> checks = List.of( + this::oldIndicesCheck, + this::ignoredOldIndicesCheck, + this::frozenIndexSettingCheck, + this::translogRetentionSettingCheck, + this::checkIndexDataPath, + this::storeTypeSettingCheck, + this::deprecatedCamelCasePattern, + this::legacyRoutingSettingCheck + ); - public IndexDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver, Map> indexToTransformIds) { + public IndexDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver) { this.indexNameExpressionResolver = indexNameExpressionResolver; - this.indexToTransformIds = indexToTransformIds; } @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { Map> indexSettingsIssues = new HashMap<>(); String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + Map> indexToTransformIds = indexToTransformIds(precomputedData.transformConfigs()); for (String concreteIndex : concreteIndexNames) { IndexMetadata indexMetadata = clusterState.getMetadata().index(concreteIndex); - List singleIndexIssues = filterChecks(indexSettingsChecks(), c -> c.apply(indexMetadata, clusterState)); + List singleIndexIssues = checks.stream() + .map(c -> c.apply(indexMetadata, clusterState, indexToTransformIds)) + .filter(Objects::nonNull) + .toList(); if (singleIndexIssues.isEmpty() == false) { indexSettingsIssues.put(concreteIndex, singleIndexIssues); } @@ -64,73 +85,131 @@ public Map> check(ClusterState clusterState, Depr return indexSettingsIssues; } - private List> indexSettingsChecks() { - return List.of( - this::oldIndicesCheck, - this::ignoredOldIndicesCheck, - IndexDeprecationChecker::translogRetentionSettingCheck, - IndexDeprecationChecker::checkIndexDataPath, - IndexDeprecationChecker::storeTypeSettingCheck, - IndexDeprecationChecker::frozenIndexSettingCheck, - IndexDeprecationChecker::deprecatedCamelCasePattern, - IndexDeprecationChecker::legacyRoutingSettingCheck - ); - } - @Override public String getName() { return NAME; } - private DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue oldIndicesCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> indexToTransformIds + ) { // TODO: this check needs to be revised. It's trivially true right now. IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 8.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), - false, - meta(indexMetadata) + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { + List cldrIncompatibleFieldMappings = new ArrayList<>(); + fieldLevelMappingIssue( + indexMetadata, + (mappingMetadata, sourceAsMap) -> cldrIncompatibleFieldMappings.addAll( + findInPropertiesRecursively( + mappingMetadata.type(), + sourceAsMap, + this::isDateFieldWithCompatFormatPattern, + this::cldrIncompatibleFormatPattern, + "", + "" + ) + ) ); + + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html" + + "#breaking_90_transform_destination_index", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to 9.0. The following transforms are " + + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " + + "transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else if (cldrIncompatibleFieldMappings.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible date format patterns in old index", + "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23", + "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. " + + "Manual reindexing is required. " + + String.join(", ", cldrIncompatibleFieldMappings), + false, + null + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } } return null; } - private Map meta(IndexMetadata indexMetadata) { - var transforms = indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of()); - if (transforms.isEmpty()) { - return Map.of("reindex_required", true); - } else { - return Map.of("reindex_required", true, "transform_ids", transforms); - } + private List transformIdsForIndex(IndexMetadata indexMetadata, Map> indexToTransformIds) { + return indexToTransformIds.getOrDefault(indexMetadata.getIndex().getName(), List.of()); } - private DeprecationIssue ignoredOldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue ignoredOldIndicesCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> indexToTransformIds + ) { IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true) && isNotDataStreamIndex(indexMetadata, clusterState)) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < 8.0 Has Been Ignored", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in 9.0", - false, - meta(indexMetadata) - ); + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html" + + "#breaking_90_transform_destination_index", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in 9.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < 8.0 has been ignored", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in 9.0", + false, + Map.of("reindex_required", true) + ); + } } return null; } - private static boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) { + private boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) { return clusterState.metadata().findDataStreams(indexMetadata.getIndex().getName()).isEmpty(); } - private static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue translogRetentionSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { final boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings()); if (softDeletesEnabled) { if (IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexMetadata.getSettings()) @@ -157,7 +236,7 @@ private static DeprecationIssue translogRetentionSettingCheck(IndexMetadata inde return null; } - private static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState, Map> ignored) { if (IndexMetadata.INDEX_DATA_PATH_SETTING.exists(indexMetadata.getSettings())) { final String message = String.format( Locale.ROOT, @@ -172,7 +251,11 @@ private static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, return null; } - private static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue storeTypeSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { final String storeType = IndexModule.INDEX_STORE_TYPE_SETTING.get(indexMetadata.getSettings()); if (IndexModule.Type.SIMPLEFS.match(storeType)) { return new DeprecationIssue( @@ -189,7 +272,11 @@ private static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadat return null; } - private static DeprecationIssue frozenIndexSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue frozenIndexSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { Boolean isIndexFrozen = FrozenEngine.INDEX_FROZEN.get(indexMetadata.getSettings()); if (Boolean.TRUE.equals(isIndexFrozen)) { String indexName = indexMetadata.getIndex().getName(); @@ -207,7 +294,11 @@ private static DeprecationIssue frozenIndexSettingCheck(IndexMetadata indexMetad return null; } - private static DeprecationIssue legacyRoutingSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue legacyRoutingSettingCheck( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(indexMetadata.getSettings()); if (deprecatedSettings.isEmpty()) { return null; @@ -223,7 +314,7 @@ private static DeprecationIssue legacyRoutingSettingCheck(IndexMetadata indexMet ); } - private static void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer> checker) { + private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer> checker) { if (indexMetadata.mapping() != null) { Map sourceAsMap = indexMetadata.mapping().sourceAsMap(); checker.accept(indexMetadata.mapping(), sourceAsMap); @@ -241,7 +332,7 @@ private static void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsum * @return a list of issues found in fields */ @SuppressWarnings("unchecked") - private static List findInPropertiesRecursively( + private List findInPropertiesRecursively( String type, Map parentMap, Function, Boolean> predicate, @@ -295,7 +386,11 @@ private static List findInPropertiesRecursively( return issues; } - private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata, ClusterState clusterState) { + private DeprecationIssue deprecatedCamelCasePattern( + IndexMetadata indexMetadata, + ClusterState clusterState, + Map> ignored + ) { List fields = new ArrayList<>(); fieldLevelMappingIssue( indexMetadata, @@ -303,8 +398,8 @@ private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMe findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, - IndexDeprecationChecker::isDateFieldWithCamelCasePattern, - IndexDeprecationChecker::changeFormatToSnakeCase, + this::isDateFieldWithCamelCasePattern, + this::changeFormatToSnakeCase, "", "" ) @@ -325,7 +420,25 @@ private static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMe return null; } - private static boolean isDateFieldWithCamelCasePattern(Map property) { + private boolean isDateFieldWithCompatFormatPattern(Map property) { + if ("date".equals(property.get("type")) && property.containsKey("format")) { + String[] patterns = DateFormatter.splitCombinedPatterns((String) property.get("format")); + for (String pattern : patterns) { + if (DateUtils.containsCompatOnlyDateFormat(pattern)) { + return true; + } + } + } + return false; + } + + private String cldrIncompatibleFormatPattern(String type, Map.Entry entry) { + Map value = (Map) entry.getValue(); + final String formatFieldValue = (String) value.get("format"); + return "Field [" + entry.getKey() + "] with format pattern [" + formatFieldValue + "]."; + } + + private boolean isDateFieldWithCamelCasePattern(Map property) { if ("date".equals(property.get("type")) && property.containsKey("format")) { String[] patterns = DateFormatter.splitCombinedPatterns((String) property.get("format")); for (String pattern : patterns) { @@ -336,7 +449,7 @@ private static boolean isDateFieldWithCamelCasePattern(Map property) { return false; } - private static String changeFormatToSnakeCase(String type, Map.Entry entry) { + private String changeFormatToSnakeCase(String type, Map.Entry entry) { Map value = (Map) entry.getValue(); final String formatFieldValue = (String) value.get("format"); String[] patterns = DateFormatter.splitCombinedPatterns(formatFieldValue); @@ -352,4 +465,14 @@ private static String changeFormatToSnakeCase(String type, Map.Entry entry sb.deleteCharAt(sb.length() - 1); return sb.toString(); } + + private Map> indexToTransformIds(List transformConfigs) { + return transformConfigs.stream() + .collect( + Collectors.groupingBy( + config -> config.getDestination().getIndex(), + Collectors.mapping(TransformConfig::getId, Collectors.toList()) + ) + ); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java deleted file mode 100644 index f7aba6491dfd2..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/LegacyIndexTemplateDeprecationChecker.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; -import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; - -/** - * Checks the legacy index templates for deprecation warnings. - */ -public class LegacyIndexTemplateDeprecationChecker implements ResourceDeprecationChecker { - - public static final String NAME = "legacy_templates"; - private static final List> CHECKS = List.of( - LegacyIndexTemplateDeprecationChecker::checkIndexTemplates - ); - - /** - * @param clusterState The cluster state provided for the checker - * @return the name of the data streams that have violated the checks with their respective warnings. - */ - @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { - var templates = clusterState.metadata().templates().entrySet(); - if (templates.isEmpty()) { - return Map.of(); - } - Map> issues = new HashMap<>(); - for (Map.Entry entry : templates) { - String name = entry.getKey(); - IndexTemplateMetadata template = entry.getValue(); - - List issuesForSingleIndexTemplate = filterChecks(CHECKS, c -> c.apply(template)); - if (issuesForSingleIndexTemplate.isEmpty() == false) { - issues.put(name, issuesForSingleIndexTemplate); - } - } - return issues.isEmpty() ? Map.of() : issues; - } - - static DeprecationIssue checkIndexTemplates(IndexTemplateMetadata indexTemplateMetadata) { - List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(indexTemplateMetadata.settings()); - if (deprecatedSettings.isEmpty()) { - return null; - } - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - DEPRECATION_MESSAGE, - DEPRECATION_HELP_URL, - "One or more of your legacy index templates is configured with 'index.routing.allocation.*.data' settings. " - + DEPRECATION_COMMON_DETAIL, - false, - DeprecationIssue.createMetaMapForRemovableSettings(deprecatedSettings) - ); - } - - @Override - public String getName() { - return NAME; - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java new file mode 100644 index 0000000000000..a2e9ed12a2298 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecker.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ThreadedActionListener; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Retrieves the individual node checks and reduces them to a list of deprecation warnings + */ +public class NodeDeprecationChecker { + + private static final Logger logger = LogManager.getLogger(NodeDeprecationChecker.class); + private final ThreadPool threadPool; + + public NodeDeprecationChecker(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void check(Client client, ActionListener> listener) { + NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.DEPRECATION_ORIGIN, + NodesDeprecationCheckAction.INSTANCE, + nodeDepReq, + new ThreadedActionListener<>(threadPool.generic(), listener.delegateFailureAndWrap((l, response) -> { + if (response.hasFailures()) { + List failedNodeIds = response.failures() + .stream() + .map(failure -> failure.nodeId() + ": " + failure.getMessage()) + .collect(Collectors.toList()); + logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); + for (FailedNodeException failure : response.failures()) { + logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); + } + } + l.onResponse(reduceToDeprecationIssues(response)); + })) + ); + } + + /** + * This method rolls up DeprecationIssues that are identical but on different nodes. It also rolls up DeprecationIssues that are + * identical (and on different nodes) except that they differ in the removable settings listed in their meta object. We roll these up + * by taking the intersection of all removable settings in otherwise identical DeprecationIssues. That way we don't claim that a + * setting can be automatically removed if any node has it in its elasticsearch.yml. + * @param response the response that contains the deprecation issues of single nodes + * @return a list of deprecation issues grouped accordingly. + */ + static List reduceToDeprecationIssues(NodesDeprecationCheckResponse response) { + // A collection whose values are lists of DeprecationIssues that differ only by meta values (if that): + Collection>> issuesToMerge = getDeprecationIssuesThatDifferOnlyByMeta(response.getNodes()); + // A map of DeprecationIssues (containing only the intersection of removable settings) to the nodes they are seen on + Map> issueToListOfNodesMap = getMergedIssuesToNodesMap(issuesToMerge); + + return issueToListOfNodesMap.entrySet().stream().map(entry -> { + DeprecationIssue issue = entry.getKey(); + String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; + return new DeprecationIssue( + issue.getLevel(), + issue.getMessage(), + issue.getUrl(), + details + "(nodes impacted: " + entry.getValue() + ")", + issue.isResolveDuringRollingUpgrade(), + issue.getMeta() + ); + }).collect(Collectors.toList()); + } + + /* + * This method pulls all the DeprecationIssues from the given nodeResponses, and buckets them into lists of DeprecationIssues that + * differ at most by meta values (if that). The returned tuples also contain the node name the deprecation issue was found on. If all + * nodes in the cluster were configured identically then all tuples in a list will differ only by the node name. + */ + private static Collection>> getDeprecationIssuesThatDifferOnlyByMeta( + List nodeResponses + ) { + Map>> issuesToMerge = new HashMap<>(); + for (NodesDeprecationCheckAction.NodeResponse resp : nodeResponses) { + for (DeprecationIssue issue : resp.getDeprecationIssues()) { + issuesToMerge.computeIfAbsent( + new DeprecationIssue( + issue.getLevel(), + issue.getMessage(), + issue.getUrl(), + issue.getDetails(), + issue.isResolveDuringRollingUpgrade(), + null // Intentionally removing meta from the key so that it's not taken into account for equality + ), + (key) -> new ArrayList<>() + ).add(new Tuple<>(issue, resp.getNode().getName())); + } + } + return issuesToMerge.values(); + } + + /* + * At this point we have one DeprecationIssue per node for a given deprecation. This method rolls them up into a single DeprecationIssue + * with a list of nodes that they appear on. If two DeprecationIssues on two different nodes differ only by the set of removable + * settings (i.e. they have different elasticsearch.yml configurations) then this method takes the intersection of those settings when + * it rolls them up. + */ + private static Map> getMergedIssuesToNodesMap( + Collection>> issuesToMerge + ) { + Map> issueToListOfNodesMap = new HashMap<>(); + for (List> similarIssues : issuesToMerge) { + DeprecationIssue leastCommonDenominator = DeprecationIssue.getIntersectionOfRemovableSettings( + similarIssues.stream().map(Tuple::v1).toList() + ); + issueToListOfNodesMap.computeIfAbsent(leastCommonDenominator, (key) -> new ArrayList<>()) + .addAll(similarIssues.stream().map(Tuple::v2).toList()); + } + return issueToListOfNodesMap; + } + +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index 0d5863e42bee8..726c381eb1f85 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -42,6 +42,61 @@ public class NodeDeprecationChecks { + // Visible for testing + static final List< + NodeDeprecationCheck> SINGLE_NODE_CHECKS = List.of( + NodeDeprecationChecks::checkMultipleDataPaths, + NodeDeprecationChecks::checkDataPathsList, + NodeDeprecationChecks::checkSharedDataPathSetting, + NodeDeprecationChecks::checkReservedPrefixedRealmNames, + NodeDeprecationChecks::checkSingleDataNodeWatermarkSetting, + NodeDeprecationChecks::checkExporterUseIngestPipelineSettings, + NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting, + NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting, + NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, + NodeDeprecationChecks::checkMonitoringSettingHistoryDuration, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecovery, + NodeDeprecationChecks::checkMonitoringSettingCollectIndices, + NodeDeprecationChecks::checkMonitoringSettingCollectCcrTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectEnrichStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexRecoveryStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectIndexStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectMlJobStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectNodeStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingCollectClusterStatsTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersHost, + NodeDeprecationChecks::checkMonitoringSettingExportersBulkTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersConnectionTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersConnectionReadTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersAuthUsername, + NodeDeprecationChecks::checkMonitoringSettingExportersAuthPass, + NodeDeprecationChecks::checkMonitoringSettingExportersSSL, + NodeDeprecationChecks::checkMonitoringSettingExportersProxyBase, + NodeDeprecationChecks::checkMonitoringSettingExportersSniffEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersHeaders, + NodeDeprecationChecks::checkMonitoringSettingExportersTemplateTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersMasterTimeout, + NodeDeprecationChecks::checkMonitoringSettingExportersEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersType, + NodeDeprecationChecks::checkMonitoringSettingExportersAlertsEnabled, + NodeDeprecationChecks::checkMonitoringSettingExportersAlertsBlacklist, + NodeDeprecationChecks::checkMonitoringSettingExportersIndexNameTimeFormat, + NodeDeprecationChecks::checkMonitoringSettingDecommissionAlerts, + NodeDeprecationChecks::checkMonitoringSettingEsCollectionEnabled, + NodeDeprecationChecks::checkMonitoringSettingCollectionEnabled, + NodeDeprecationChecks::checkMonitoringSettingCollectionInterval, + NodeDeprecationChecks::checkScriptContextCache, + NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, + NodeDeprecationChecks::checkScriptContextCacheSizeSetting, + NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, + NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting, + NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, + NodeDeprecationChecks::checkEqlEnabledSetting, + NodeDeprecationChecks::checkNodeAttrData, + NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, + NodeDeprecationChecks::checkTracingApmSettings + ); + static DeprecationIssue checkDeprecatedSetting( final Settings clusterSettings, final Settings nodeSettings, @@ -78,15 +133,6 @@ private static Map createMetaMapForRemovableSettings(boolean can return canAutoRemoveSetting ? DeprecationIssue.createMetaMapForRemovableSettings(removableSettings) : null; } - static DeprecationIssue checkRemovedSetting( - final Settings clusterSettings, - final Settings nodeSettings, - final Setting removedSetting, - final String url - ) { - return checkRemovedSetting(clusterSettings, nodeSettings, removedSetting, url, null, DeprecationIssue.Level.CRITICAL); - } - static DeprecationIssue checkRemovedSetting( final Settings clusterSettings, final Settings nodeSettings, @@ -1035,4 +1081,9 @@ static DeprecationIssue checkTracingApmSettings( DeprecationIssue.Level.CRITICAL ); } + + @FunctionalInterface + public interface NodeDeprecationCheck { + R apply(A first, B second, C third, D fourth); + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java index 71b9903f69f86..daa3514e3b989 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ResourceDeprecationChecker.java @@ -24,8 +24,14 @@ public interface ResourceDeprecationChecker { * This runs the checks for the current deprecation checker. * * @param clusterState The cluster state provided for the checker + * @param request The deprecation request that triggered this check + * @param precomputedData Data that have been remotely retrieved and might be useful in the checks */ - Map> check(ClusterState clusterState, DeprecationInfoAction.Request request); + Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ); /** * @return The name of the checker diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java index 235209243ee58..d08918b562667 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java @@ -17,11 +17,14 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestDeprecationInfoAction extends BaseRestHandler { + private static final Set SUPPORTED_CAPABILITIES = Set.of("data_streams", "ilm_policies", "templates"); + @Override public List routes() { return List.of( @@ -53,4 +56,9 @@ private static RestChannelConsumer handleGet(final RestRequest request, NodeClie ); return channel -> client.execute(DeprecationInfoAction.INSTANCE, infoRequest, new RestToXContentListener<>(channel)); } + + @Override + public Set supportedCapabilities() { + return SUPPORTED_CAPABILITIES; + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java index 5a451a9613797..ff0ff982d11bf 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationChecker.java @@ -19,9 +19,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_HELP_URL; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_MESSAGE; @@ -32,20 +32,34 @@ public class TemplateDeprecationChecker implements ResourceDeprecationChecker { public static final String NAME = "templates"; - private static final List> INDEX_TEMPLATE_CHECKS = List.of( - TemplateDeprecationChecker::checkLegacyTiersInIndexTemplate + private final List> indexTemplateChecks = List.of( + this::checkLegacyTiersInIndexTemplate ); - private static final List> COMPONENT_TEMPLATE_CHECKS = List.of( - TemplateDeprecationChecker::checkSourceModeInComponentTemplates, - TemplateDeprecationChecker::checkLegacyTiersInComponentTemplates + private final List> componentTemplateChecks = List.of( + this::checkSourceModeInComponentTemplates, + this::checkLegacyTiersInComponentTemplates ); /** * @param clusterState The cluster state provided for the checker + * @param request not used yet in these checks + * @param precomputedData not used yet in these checks * @return the name of the data streams that have violated the checks with their respective warnings. */ @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check(clusterState); + } + + /** + * @param clusterState The cluster state provided for the checker + * @return the name of the data streams that have violated the checks with their respective warnings. + */ + Map> check(ClusterState clusterState) { var indexTemplates = clusterState.metadata().templatesV2().entrySet(); var componentTemplates = clusterState.metadata().componentTemplates().entrySet(); if (indexTemplates.isEmpty() && componentTemplates.isEmpty()) { @@ -56,7 +70,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); ComposableIndexTemplate template = entry.getValue(); - List issuesForSingleIndexTemplate = filterChecks(INDEX_TEMPLATE_CHECKS, c -> c.apply(template)); + List issuesForSingleIndexTemplate = indexTemplateChecks.stream() + .map(c -> c.apply(template)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleIndexTemplate.isEmpty() == false) { issues.computeIfAbsent(name, ignored -> new ArrayList<>()).addAll(issuesForSingleIndexTemplate); } @@ -65,7 +82,10 @@ public Map> check(ClusterState clusterState, Depr String name = entry.getKey(); ComponentTemplate template = entry.getValue(); - List issuesForSingleIndexTemplate = filterChecks(COMPONENT_TEMPLATE_CHECKS, c -> c.apply(template)); + List issuesForSingleIndexTemplate = componentTemplateChecks.stream() + .map(c -> c.apply(template)) + .filter(Objects::nonNull) + .toList(); if (issuesForSingleIndexTemplate.isEmpty() == false) { issues.computeIfAbsent(name, ignored -> new ArrayList<>()).addAll(issuesForSingleIndexTemplate); } @@ -73,7 +93,7 @@ public Map> check(ClusterState clusterState, Depr return issues.isEmpty() ? Map.of() : issues; } - static DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate composableIndexTemplate) { + private DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate composableIndexTemplate) { Template template = composableIndexTemplate.template(); if (template != null) { List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(template.settings()); @@ -93,7 +113,7 @@ static DeprecationIssue checkLegacyTiersInIndexTemplate(ComposableIndexTemplate return null; } - static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate template) { + private DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate template) { if (template.template().mappings() != null) { var sourceAsMap = (Map) XContentHelper.convertToMap(template.template().mappings().uncompressed(), true).v2().get("_doc"); if (sourceAsMap != null) { @@ -102,9 +122,9 @@ static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate te if (sourceMap.containsKey("mode")) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, + SourceFieldMapper.DEPRECATION_WARNING_TITLE, + "https://ela.st/migrate-source-mode", SourceFieldMapper.DEPRECATION_WARNING, - "https://github.com/elastic/elasticsearch/pull/117172", - null, false, null ); @@ -115,7 +135,7 @@ static DeprecationIssue checkSourceModeInComponentTemplates(ComponentTemplate te return null; } - static DeprecationIssue checkLegacyTiersInComponentTemplates(ComponentTemplate componentTemplate) { + private DeprecationIssue checkLegacyTiersInComponentTemplates(ComponentTemplate componentTemplate) { Template template = componentTemplate.template(); List deprecatedSettings = LegacyTiersDetection.getDeprecatedFilteredAllocationSettings(template.settings()); if (deprecatedSettings.isEmpty()) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java deleted file mode 100644 index 57c4fae960854..0000000000000 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; -import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; - -import java.util.ArrayList; -import java.util.List; - -class TransformDeprecationChecker implements DeprecationChecker { - - public static final String TRANSFORM_DEPRECATION_KEY = "transform_settings"; - private final List transformConfigs; - - TransformDeprecationChecker(List transformConfigs) { - this.transformConfigs = transformConfigs; - } - - @Override - public boolean enabled(Settings settings) { - // always enabled - return true; - } - - @Override - public void check(Components components, ActionListener deprecationIssueListener) { - ActionListener.completeWith(deprecationIssueListener, () -> { - List allIssues = new ArrayList<>(); - for (var config : transformConfigs) { - allIssues.addAll(config.checkForDeprecations(components.xContentRegistry())); - } - return new CheckResult(getName(), allIssues); - }); - } - - @Override - public String getName() { - return TRANSFORM_DEPRECATION_KEY; - } -} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 886eddf82149e..c30d8829c23f3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -6,12 +6,11 @@ */ package org.elasticsearch.xpack.deprecation; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.client.internal.OriginSettingClient; @@ -19,14 +18,22 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.action.util.PageParams; @@ -35,24 +42,30 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; - public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction< DeprecationInfoAction.Request, DeprecationInfoAction.Response> { - private static final DeprecationChecker ML_CHECKER = new MlDeprecationChecker(); - private static final Logger logger = LogManager.getLogger(TransportDeprecationInfoAction.class); + public static final Setting> SKIP_DEPRECATIONS_SETTING = Setting.stringListSetting( + "deprecation.skip_deprecated_settings", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final List PLUGIN_CHECKERS = List.of(new MlDeprecationChecker()); private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; private final Settings settings; private final NamedXContentRegistry xContentRegistry; private volatile List skipTheseDeprecations; + private final NodeDeprecationChecker nodeDeprecationChecker; + private final ClusterDeprecationChecker clusterDeprecationChecker; + private final List resourceDeprecationCheckers; @Inject public TransportDeprecationInfoAction( @@ -79,10 +92,17 @@ public TransportDeprecationInfoAction( this.indexNameExpressionResolver = indexNameExpressionResolver; this.settings = settings; this.xContentRegistry = xContentRegistry; - skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings); + skipTheseDeprecations = SKIP_DEPRECATIONS_SETTING.get(settings); + nodeDeprecationChecker = new NodeDeprecationChecker(threadPool); + clusterDeprecationChecker = new ClusterDeprecationChecker(xContentRegistry); + resourceDeprecationCheckers = List.of( + new IndexDeprecationChecker(indexNameExpressionResolver), + new DataStreamDeprecationChecker(indexNameExpressionResolver), + new TemplateDeprecationChecker(), + new IlmPolicyDeprecationChecker() + ); // Safe to register this here because it happens synchronously before the cluster service is started: - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(DeprecationChecks.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); } private void setSkipDeprecations(List skipDeprecations) { @@ -102,56 +122,222 @@ protected final void masterOperation( ClusterState state, final ActionListener listener ) { - NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); - ClientHelper.executeAsyncWithOrigin( - client, - ClientHelper.DEPRECATION_ORIGIN, - NodesDeprecationCheckAction.INSTANCE, - nodeDepReq, - listener.delegateFailureAndWrap((l, response) -> { - if (response.hasFailures()) { - List failedNodeIds = response.failures() - .stream() - .map(failure -> failure.nodeId() + ": " + failure.getMessage()) - .collect(Collectors.toList()); - logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); - for (FailedNodeException failure : response.failures()) { - logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); - } - } - transformConfigs(l.delegateFailureAndWrap((ll, transformConfigs) -> { - DeprecationChecker.Components components = new DeprecationChecker.Components( - xContentRegistry, - settings, - new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) - ); - pluginSettingIssues( - List.of(ML_CHECKER, new TransformDeprecationChecker(transformConfigs)), - components, - new ThreadedActionListener<>( - client.threadPool().generic(), - ll.map( - deprecationIssues -> DeprecationInfoAction.Response.from( - state, - indexNameExpressionResolver, - request, - response, - CLUSTER_SETTINGS_CHECKS, - deprecationIssues, - skipTheseDeprecations, - List.of( - new IndexDeprecationChecker(indexNameExpressionResolver, indexToTransformIds(transformConfigs)), - new DataStreamDeprecationChecker(indexNameExpressionResolver), - new TemplateDeprecationChecker(), - new IlmPolicyDeprecationChecker() - ) - ) - ) - ) - ); - })); - }) + PrecomputedData precomputedData = new PrecomputedData(); + try (var refs = new RefCountingListener(checkAndCreateResponse(state, request, precomputedData, listener))) { + nodeDeprecationChecker.check(client, refs.acquire(precomputedData::setOnceNodeSettingsIssues)); + transformConfigs(refs.acquire(precomputedData::setOnceTransformConfigs)); + DeprecationChecker.Components components = new DeprecationChecker.Components( + xContentRegistry, + settings, + new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) + ); + pluginSettingIssues(PLUGIN_CHECKERS, components, refs.acquire(precomputedData::setOncePluginIssues)); + } + } + + /** + * This is the function that does the bulk of the logic of combining the necessary dependencies together, including the cluster state, + * the precalculated information in {@code context} with the remaining checkers such as the cluster setting checker and the resource + * checkers.This function will run a significant part of the checks and build out the final list of issues that exist in the + * cluster. Because of that, it's important that it does not run in the transport thread that's why it's combined with + * {@link #executeInGenericThreadpool(ActionListener)}. + * + * @param state The cluster state + * @param request The originating request containing the index expressions to evaluate + * @param precomputedData Data from remote requests necessary to construct the response + * @param responseListener The listener expecting the {@link DeprecationInfoAction.Response} + * @return The listener that should be executed after all the remote requests have completed and the {@link PrecomputedData} + * is initialised. + */ + public ActionListener checkAndCreateResponse( + ClusterState state, + DeprecationInfoAction.Request request, + PrecomputedData precomputedData, + ActionListener responseListener + ) { + return executeInGenericThreadpool( + ActionListener.running( + () -> responseListener.onResponse( + checkAndCreateResponse( + state, + indexNameExpressionResolver, + request, + skipTheseDeprecations, + clusterDeprecationChecker, + resourceDeprecationCheckers, + precomputedData + ) + ) + ) + ); + } + + /** + * This is the function that does the bulk of the logic of combining the necessary dependencies together, including the cluster state, + * the precalculated information in {@code context} with the remaining checkers such as the cluster setting checker and the resource + * checkers.This function will run a significant part of the checks and build out the final list of issues that exist in the + * cluster. It's important that it does not run in the transport thread that's why it's combined with + * {@link #checkAndCreateResponse(ClusterState, DeprecationInfoAction.Request, PrecomputedData, ActionListener)}. We keep this separated + * for testing purposes. + * + * @param state The cluster state + * @param indexNameExpressionResolver Used to resolve indices into their concrete names + * @param request The originating request containing the index expressions to evaluate + * @param skipTheseDeprecatedSettings the settings that will be removed from cluster metadata and the index metadata of all the + * indexes specified by indexNames + * @param clusterDeprecationChecker The checker that provides the cluster settings deprecations warnings + * @param resourceDeprecationCheckers these are checkers that take as input the cluster state and return a map from resource type + * to issues grouped by the resource name. + * @param precomputedData data from remote requests necessary to construct the response + * @return The list of deprecation issues found in the cluster + */ + static DeprecationInfoAction.Response checkAndCreateResponse( + ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver, + DeprecationInfoAction.Request request, + List skipTheseDeprecatedSettings, + ClusterDeprecationChecker clusterDeprecationChecker, + List resourceDeprecationCheckers, + PrecomputedData precomputedData + ) { + assert Transports.assertNotTransportThread("walking mappings in indexSettingsChecks is expensive"); + // Allow system index access here to prevent deprecation warnings when we call this API + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, request); + ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); + List clusterSettingsIssues = clusterDeprecationChecker.check( + stateWithSkippedSettingsRemoved, + precomputedData.transformConfigs() + ); + + Map>> resourceDeprecationIssues = new HashMap<>(); + for (ResourceDeprecationChecker resourceDeprecationChecker : resourceDeprecationCheckers) { + Map> issues = resourceDeprecationChecker.check( + stateWithSkippedSettingsRemoved, + request, + precomputedData + ); + if (issues.isEmpty() == false) { + resourceDeprecationIssues.put(resourceDeprecationChecker.getName(), issues); + } + } + + return new DeprecationInfoAction.Response( + clusterSettingsIssues, + precomputedData.nodeSettingsIssues(), + resourceDeprecationIssues, + precomputedData.pluginIssues() + ); + } + + /** + * This class holds the results of remote requests. These can be either checks that require remote requests such as + * {@code nodeSettingsIssues} and {@code pluginIssues} or metadata needed for more than one types of checks such as + * {@code transformConfigs}. + */ + public static class PrecomputedData { + private final SetOnce> nodeSettingsIssues = new SetOnce<>(); + private final SetOnce>> pluginIssues = new SetOnce<>(); + private final SetOnce> transformConfigs = new SetOnce<>(); + + public void setOnceNodeSettingsIssues(List nodeSettingsIssues) { + this.nodeSettingsIssues.set(nodeSettingsIssues); + } + + public void setOncePluginIssues(Map> pluginIssues) { + this.pluginIssues.set(pluginIssues); + } + + public void setOnceTransformConfigs(List transformConfigs) { + this.transformConfigs.set(transformConfigs); + } + + public List nodeSettingsIssues() { + return nodeSettingsIssues.get(); + } + + public Map> pluginIssues() { + return pluginIssues.get(); + } + + public List transformConfigs() { + return transformConfigs.get(); + } + } + + /** + * Removes the skipped settings from the selected indices and the component and index templates. + * @param state The cluster state to modify + * @param indexNames The names of the indexes whose settings need to be filtered + * @param skipTheseDeprecatedSettings The settings that will be removed from cluster metadata and the index metadata of all the + * indexes specified by indexNames + * @return A modified cluster state with the given settings removed + */ + private static ClusterState removeSkippedSettings(ClusterState state, String[] indexNames, List skipTheseDeprecatedSettings) { + // Short-circuit, no need to reconstruct the cluster state if there are no settings to remove + if (skipTheseDeprecatedSettings == null || skipTheseDeprecatedSettings.isEmpty()) { + return state; + } + ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()); + metadataBuilder.transientSettings( + metadataBuilder.transientSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) + ); + metadataBuilder.persistentSettings( + metadataBuilder.persistentSettings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) ); + Map indicesBuilder = new HashMap<>(state.getMetadata().indices()); + for (String indexName : indexNames) { + IndexMetadata indexMetadata = state.getMetadata().index(indexName); + IndexMetadata.Builder filteredIndexMetadataBuilder = new IndexMetadata.Builder(indexMetadata); + Settings filteredSettings = indexMetadata.getSettings() + .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false); + filteredIndexMetadataBuilder.settings(filteredSettings); + indicesBuilder.put(indexName, filteredIndexMetadataBuilder.build()); + } + metadataBuilder.componentTemplates(state.metadata().componentTemplates().entrySet().stream().map(entry -> { + String templateName = entry.getKey(); + ComponentTemplate componentTemplate = entry.getValue(); + Template template = componentTemplate.template(); + if (template.settings() == null || template.settings().isEmpty()) { + return Tuple.tuple(templateName, componentTemplate); + } + return Tuple.tuple( + templateName, + new ComponentTemplate( + Template.builder(template) + .settings(template.settings().filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false)) + .build(), + componentTemplate.version(), + componentTemplate.metadata(), + componentTemplate.deprecated() + ) + ); + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + metadataBuilder.indexTemplates(state.metadata().templatesV2().entrySet().stream().map(entry -> { + String templateName = entry.getKey(); + ComposableIndexTemplate indexTemplate = entry.getValue(); + Template template = indexTemplate.template(); + if (template == null || template.settings() == null || template.settings().isEmpty()) { + return Tuple.tuple(templateName, indexTemplate); + } + return Tuple.tuple( + templateName, + indexTemplate.toBuilder() + .template( + Template.builder(indexTemplate.template()) + .settings( + indexTemplate.template() + .settings() + .filter(setting -> Regex.simpleMatch(skipTheseDeprecatedSettings, setting) == false) + ) + ) + .build() + ); + }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + + metadataBuilder.indices(indicesBuilder); + clusterStateBuilder.metadata(metadataBuilder); + return clusterStateBuilder.build(); } static void pluginSettingIssues( @@ -192,34 +378,21 @@ private void transformConfigs(PageParams currentPage, ActionListener( - threadPool.generic(), - currentPageListener.delegateFailureAndWrap((delegate, getTransformConfigResponse) -> { - var currentPageOfConfigs = getTransformConfigResponse.getTransformConfigurations().stream(); - var currentPageSize = currentPage.getFrom() + currentPage.getSize(); - var totalTransformConfigCount = getTransformConfigResponse.getTransformConfigurationCount(); - if (totalTransformConfigCount >= currentPageSize) { - var nextPage = new PageParams(currentPageSize, PageParams.DEFAULT_SIZE); - transformConfigs( - nextPage, - delegate.map(nextPageOfConfigs -> Stream.concat(currentPageOfConfigs, nextPageOfConfigs)) - ); - } else { - delegate.onResponse(currentPageOfConfigs); - } - }) - ) + executeInGenericThreadpool(currentPageListener.delegateFailureAndWrap((delegate, getTransformConfigResponse) -> { + var currentPageOfConfigs = getTransformConfigResponse.getTransformConfigurations().stream(); + var currentPageSize = currentPage.getFrom() + currentPage.getSize(); + var totalTransformConfigCount = getTransformConfigResponse.getTransformConfigurationCount(); + if (totalTransformConfigCount >= currentPageSize) { + var nextPage = new PageParams(currentPageSize, PageParams.DEFAULT_SIZE); + transformConfigs(nextPage, delegate.map(nextPageOfConfigs -> Stream.concat(currentPageOfConfigs, nextPageOfConfigs))); + } else { + delegate.onResponse(currentPageOfConfigs); + } + })) ); } - private Map> indexToTransformIds(List transformConfigs) { - return transformConfigs.stream() - .collect( - Collectors.groupingBy( - config -> config.getDestination().getIndex(), - Collectors.mapping(TransformConfig::getId, Collectors.toList()) - ) - ); + private ActionListener executeInGenericThreadpool(ActionListener listener) { + return new ThreadedActionListener<>(threadPool.generic(), listener); } - } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java index 745f5e7ae8959..befe0bd6b41a4 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -36,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; @@ -75,10 +76,10 @@ public TransportNodeDeprecationCheckAction( this.pluginsService = pluginsService; this.licenseState = licenseState; this.clusterInfoService = clusterInfoService; - skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings); + skipTheseDeprecations = TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.get(settings); // Safe to register this here because it happens synchronously before the cluster service is started: clusterService.getClusterSettings() - .addSettingsUpdateConsumer(DeprecationChecks.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); + .addSettingsUpdateConsumer(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING, this::setSkipDeprecations); } private void setSkipDeprecations(List skipDeprecations) { @@ -106,13 +107,13 @@ protected NodesDeprecationCheckAction.NodeResponse newNodeResponse(StreamInput i @Override protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request, Task task) { - return nodeOperation(request, DeprecationChecks.NODE_SETTINGS_CHECKS); + return nodeOperation(request, NodeDeprecationChecks.SINGLE_NODE_CHECKS); } NodesDeprecationCheckAction.NodeResponse nodeOperation( NodesDeprecationCheckAction.NodeRequest request, List< - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -130,10 +131,10 @@ NodesDeprecationCheckAction.NodeResponse nodeOperation( .metadata(Metadata.builder(metadata).transientSettings(transientSettings).persistentSettings(persistentSettings).build()) .build(); - List issues = DeprecationInfoAction.filterChecks( - nodeSettingsChecks, - (c) -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState) - ); + List issues = nodeSettingsChecks.stream() + .map(c -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState)) + .filter(Objects::nonNull) + .toList(); DeprecationIssue watermarkIssue = checkDiskLowWatermark( filteredNodeSettings, filteredClusterState.metadata().settings(), diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index d9cbf0d3f1152..3c0ef84cb6193 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -69,7 +69,7 @@ public void testOldIndicesCheck() { ); // We know that the data stream checks ignore the request. - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.size(), equalTo(1)); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); @@ -91,7 +91,7 @@ public void testOldIndicesCheckWithOnlyNewIndices() { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.size(), equalTo(0)); } @@ -137,7 +137,7 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { ) ); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } @@ -290,16 +290,64 @@ public void testOldIndicesIgnoredWarningCheck() { + "OK to remain read-only after upgrade", false, ofEntries( - entry("reindex_required", true), + entry("reindex_required", false), entry("total_backing_indices", oldIndexCount + newIndexCount), entry("ignored_indices_requiring_upgrade_count", expectedIndices.size()), entry("ignored_indices_requiring_upgrade", expectedIndices) ) ); - Map> issuesByDataStream = checker.check(clusterState, null); + Map> issuesByDataStream = checker.check(clusterState); assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldSystemDataStreamIgnored() { + // We do not want system data streams coming back in the deprecation info API + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + List allIndices = new ArrayList<>(); + Map nameToIndexMetadata = new HashMap<>(); + for (int i = 0; i < oldIndexCount; i++) { + Settings.Builder settings = settings(IndexVersion.fromId(7170099)); + + String indexName = "old-data-stream-index-" + i; + settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + + IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName) + .settings(settings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + allIndices.add(oldIndexMetadata.getIndex()); + nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); + } + for (int i = 0; i < newIndexCount; i++) { + Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + allIndices.add(newIndex); + } + DataStream dataStream = new DataStream( + randomAlphaOfLength(10), + allIndices, + randomNonNegativeLong(), + Map.of(), + true, + false, + true, + randomBoolean(), + randomFrom(IndexMode.values()), + null, + randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null), + List.of(), + randomBoolean(), + null + ); + Metadata metadata = Metadata.builder() + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + assertThat(checker.check(clusterState), equalTo(Map.of())); + } + } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java deleted file mode 100644 index 9a57450b7fad7..0000000000000 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationChecksTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.deprecation; - -import org.elasticsearch.core.Tuple; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; - -import static org.hamcrest.Matchers.equalTo; - -public class DeprecationChecksTests extends ESTestCase { - - public void testFilterChecks() { - DeprecationIssue issue = createRandomDeprecationIssue(); - int numChecksPassed = randomIntBetween(0, 5); - int numChecksFailed = 10 - numChecksPassed; - List> checks = new ArrayList<>(); - for (int i = 0; i < numChecksFailed; i++) { - checks.add(() -> issue); - } - for (int i = 0; i < numChecksPassed; i++) { - checks.add(() -> null); - } - List filteredIssues = DeprecationInfoAction.filterChecks(checks, Supplier::get); - assertThat(filteredIssues.size(), equalTo(numChecksFailed)); - } - - private static DeprecationIssue createRandomDeprecationIssue() { - String details = randomBoolean() ? randomAlphaOfLength(10) : null; - return new DeprecationIssue( - randomFrom(DeprecationIssue.Level.values()), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - details, - randomBoolean(), - randomMap(1, 5, () -> Tuple.tuple(randomAlphaOfLength(4), randomAlphaOfLength(4))) - ); - } -} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java index 28fd14abecbc1..537c3eb84a902 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java @@ -6,50 +6,17 @@ */ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; -import org.junit.Assert; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.Response.RESERVED_NAMES; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.core.IsEqual.equalTo; - public class DeprecationInfoActionResponseTests extends AbstractWireSerializingTestCase { @Override @@ -153,318 +120,11 @@ protected Writeable.Reader instanceReader() { return DeprecationInfoAction.Response::new; } - public void testFrom() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); - mapping.field("enabled", false); - mapping.endObject().endObject(); - - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("test") - .putMapping(Strings.toString(mapping)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - - DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - boolean clusterIssueFound = randomBoolean(); - boolean nodeIssueFound = randomBoolean(); - boolean indexIssueFound = randomBoolean(); - boolean dataStreamIssueFound = randomBoolean(); - boolean indexTemplateIssueFound = randomBoolean(); - boolean componentTemplateIssueFound = randomBoolean(); - boolean ilmPolicyIssueFound = randomBoolean(); - DeprecationIssue foundIssue = createTestDeprecationIssue(); - List> clusterSettingsChecks = List.of((s) -> clusterIssueFound ? foundIssue : null); - List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { - if (indexIssueFound) { - return Map.of("test", List.of(foundIssue)); - } - return Map.of(); - }), createResourceChecker("data_streams", (cs, req) -> { - if (dataStreamIssueFound) { - return Map.of("my-ds", List.of(foundIssue)); - } - return Map.of(); - }), createResourceChecker("templates", (cs, req) -> { - Map> issues = new HashMap<>(); - if (componentTemplateIssueFound) { - issues.put("my-component-template", List.of(foundIssue)); - } - if (indexTemplateIssueFound) { - issues.put("my-index-template", List.of(foundIssue)); - } - return issues; - }), createResourceChecker("ilm_policies", (cs, req) -> { - if (ilmPolicyIssueFound) { - return Map.of("my-policy", List.of(foundIssue)); - } - return Map.of(); - })); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - nodeIssueFound ? List.of(new NodesDeprecationCheckAction.NodeResponse(discoveryNode, List.of(foundIssue))) : List.of(), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of(), - resourceCheckers - ); - - if (clusterIssueFound) { - assertThat(response.getClusterSettingsIssues(), equalTo(List.of(foundIssue))); - } else { - assertThat(response.getClusterSettingsIssues(), empty()); - } - - if (nodeIssueFound) { - String details = foundIssue.getDetails() != null ? foundIssue.getDetails() + " " : ""; - DeprecationIssue mergedFoundIssue = new DeprecationIssue( - foundIssue.getLevel(), - foundIssue.getMessage(), - foundIssue.getUrl(), - details + "(nodes impacted: [" + discoveryNode.getName() + "])", - foundIssue.isResolveDuringRollingUpgrade(), - foundIssue.getMeta() - ); - assertThat(response.getNodeSettingsIssues(), equalTo(List.of(mergedFoundIssue))); - } else { - assertTrue(response.getNodeSettingsIssues().isEmpty()); - } - - if (indexIssueFound) { - assertThat(response.getIndexSettingsIssues(), equalTo(Map.of("test", List.of(foundIssue)))); - } else { - assertTrue(response.getIndexSettingsIssues().isEmpty()); - } - if (dataStreamIssueFound) { - assertThat(response.getDataStreamDeprecationIssues(), equalTo(Map.of("my-ds", List.of(foundIssue)))); - } else { - assertTrue(response.getDataStreamDeprecationIssues().isEmpty()); - } - if (ilmPolicyIssueFound) { - assertThat(response.getIlmPolicyDeprecationIssues(), equalTo(Map.of("my-policy", List.of(foundIssue)))); - } else { - assertTrue(response.getIlmPolicyDeprecationIssues().isEmpty()); - } - if (componentTemplateIssueFound == false && indexTemplateIssueFound == false) { - assertTrue(response.getTemplateDeprecationIssues().isEmpty()); - } else { - if (componentTemplateIssueFound) { - assertThat(response.getTemplateDeprecationIssues().get("my-component-template"), equalTo(List.of(foundIssue))); - } - if (indexTemplateIssueFound) { - assertThat(response.getTemplateDeprecationIssues().get("my-index-template"), equalTo(List.of(foundIssue))); - } - - } - } - - public void testFromWithMergeableNodeIssues() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); - mapping.field("enabled", false); - mapping.endObject().endObject(); - - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("test") - .putMapping(Strings.toString(mapping)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - - DiscoveryNode node1 = DiscoveryNodeUtils.builder("nodeId1") - .name("node1") - .ephemeralId("ephemeralId1") - .address("hostName1", "hostAddress1", new TransportAddress(TransportAddress.META_ADDRESS, 9300)) - .roles(Set.of()) - .build(); - DiscoveryNode node2 = DiscoveryNodeUtils.builder("nodeId2") - .name("node2") - .ephemeralId("ephemeralId2") - .address("hostName2", "hostAddress2", new TransportAddress(TransportAddress.META_ADDRESS, 9500)) - .roles(Set.of()) - .build(); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - Map metaMap1 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.1", "setting.2", "setting.3")); - Map metaMap2 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.2", "setting.3")); - DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1); - DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2); - List> clusterSettingsChecks = List.of(); - List resourceCheckers = List.of(); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - Arrays.asList( - new NodesDeprecationCheckAction.NodeResponse(node1, List.of(foundIssue1)), - new NodesDeprecationCheckAction.NodeResponse(node2, List.of(foundIssue2)) - ), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of(), - resourceCheckers - ); - - String details = foundIssue1.getDetails() != null ? foundIssue1.getDetails() + " " : ""; - DeprecationIssue mergedFoundIssue = new DeprecationIssue( - foundIssue1.getLevel(), - foundIssue1.getMessage(), - foundIssue1.getUrl(), - details + "(nodes impacted: [" + node1.getName() + ", " + node2.getName() + "])", - foundIssue1.isResolveDuringRollingUpgrade(), - foundIssue2.getMeta() - ); - assertThat(response.getNodeSettingsIssues(), equalTo(List.of(mergedFoundIssue))); - } - - public void testRemoveSkippedSettings() { - Settings.Builder settingsBuilder = settings(IndexVersion.current()); - settingsBuilder.put("some.deprecated.property", "someValue1"); - settingsBuilder.put("some.other.bad.deprecated.property", "someValue2"); - settingsBuilder.put("some.undeprecated.property", "someValue3"); - settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); - Settings inputSettings = settingsBuilder.build(); - IndexMetadata dataStreamIndexMetadata = IndexMetadata.builder("ds-test-index-1") - .settings(inputSettings) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - ComponentTemplate componentTemplate = new ComponentTemplate(Template.builder().settings(inputSettings).build(), null, null); - ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() - .template(Template.builder().settings(inputSettings)) - .build(); - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("test").settings(inputSettings).numberOfShards(1).numberOfReplicas(0)) - .put(dataStreamIndexMetadata, true) - .put(DataStream.builder("ds-test", List.of(dataStreamIndexMetadata.getIndex())).build()) - .indexTemplates(Map.of("my-index-template", indexTemplate)) - .componentTemplates(Map.of("my-component-template", componentTemplate)) - .persistentSettings(inputSettings) - .build(); - - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); - AtomicReference visibleClusterSettings = new AtomicReference<>(); - List> clusterSettingsChecks = List.of((s) -> { - visibleClusterSettings.set(s.getMetadata().settings()); - return null; - }); - AtomicReference visibleIndexSettings = new AtomicReference<>(); - AtomicReference visibleComponentTemplateSettings = new AtomicReference<>(); - AtomicReference visibleIndexTemplateSettings = new AtomicReference<>(); - AtomicInteger backingIndicesCount = new AtomicInteger(0); - List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { - for (String indexName : resolver.concreteIndexNames(cs, req)) { - visibleIndexSettings.set(cs.metadata().index(indexName).getSettings()); - } - return Map.of(); - }), createResourceChecker("data_streams", (cs, req) -> { - cs.metadata().dataStreams().values().forEach(ds -> backingIndicesCount.set(ds.getIndices().size())); - return Map.of(); - }), createResourceChecker("templates", (cs, req) -> { - cs.metadata() - .componentTemplates() - .values() - .forEach(template -> visibleComponentTemplateSettings.set(template.template().settings())); - cs.metadata().templatesV2().values().forEach(template -> visibleIndexTemplateSettings.set(template.template().settings())); - return Map.of(); - })); - - NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( - new ClusterName(randomAlphaOfLength(5)), - List.of(), - List.of() - ); - - DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); - DeprecationInfoAction.Response.from( - state, - resolver, - request, - nodeDeprecationIssues, - clusterSettingsChecks, - new HashMap<>(), // modified in the method to move transform deprecation issues into cluster_settings - List.of("some.deprecated.property", "some.other.*.deprecated.property"), - resourceCheckers - ); - - settingsBuilder = settings(IndexVersion.current()); - settingsBuilder.put("some.undeprecated.property", "someValue3"); - settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); - - Settings expectedSettings = settingsBuilder.build(); - Settings resultClusterSettings = visibleClusterSettings.get(); - Assert.assertNotNull(resultClusterSettings); - Assert.assertEquals(expectedSettings, visibleClusterSettings.get()); - - Settings resultIndexSettings = visibleIndexSettings.get(); - Assert.assertNotNull(resultIndexSettings); - Assert.assertEquals("someValue3", resultIndexSettings.get("some.undeprecated.property")); - Assert.assertEquals(resultIndexSettings.getAsList("some.undeprecated.list.property"), List.of("someValue4", "someValue5")); - Assert.assertFalse(resultIndexSettings.hasValue("some.deprecated.property")); - Assert.assertFalse(resultIndexSettings.hasValue("some.other.bad.deprecated.property")); - - assertThat(backingIndicesCount.get(), equalTo(1)); - - Assert.assertNotNull(visibleComponentTemplateSettings.get()); - Assert.assertEquals(expectedSettings, visibleComponentTemplateSettings.get()); - Assert.assertNotNull(visibleIndexTemplateSettings.get()); - Assert.assertEquals(expectedSettings, visibleIndexTemplateSettings.get()); - } - - public void testCtorFailure() { - Map> indexNames = Stream.generate(() -> randomAlphaOfLength(10)) - .limit(10) - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - Map> dataStreamNames = Stream.generate(() -> randomAlphaOfLength(10)) - .limit(10) - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - Set shouldCauseFailure = new HashSet<>(RESERVED_NAMES); - for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) { - Map> pluginSettingsIssues = randomSubsetOf(3, shouldCauseFailure).stream() - .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); - expectThrows( - ElasticsearchStatusException.class, - () -> new DeprecationInfoAction.Response( - List.of(), - List.of(), - Map.of("data_streams", dataStreamNames, "index_settings", indexNames), - pluginSettingsIssues - ) - ); - } - } - - private static DeprecationIssue createTestDeprecationIssue() { + static DeprecationIssue createTestDeprecationIssue() { return createTestDeprecationIssue(randomMap(1, 5, () -> Tuple.tuple(randomAlphaOfLength(4), randomAlphaOfLength(4)))); } - private static DeprecationIssue createTestDeprecationIssue(Map metaMap) { + static DeprecationIssue createTestDeprecationIssue(Map metaMap) { String details = randomBoolean() ? randomAlphaOfLength(10) : null; return new DeprecationIssue( randomFrom(Level.values()), @@ -476,7 +136,7 @@ private static DeprecationIssue createTestDeprecationIssue(Map m ); } - private static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seedIssue, Map metaMap) { + static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seedIssue, Map metaMap) { return new DeprecationIssue( seedIssue.getLevel(), seedIssue.getMessage(), @@ -487,27 +147,9 @@ private static DeprecationIssue createTestDeprecationIssue(DeprecationIssue seed ); } - private static List randomDeprecationIssues() { + static List randomDeprecationIssues() { return Stream.generate(DeprecationInfoActionResponseTests::createTestDeprecationIssue) .limit(randomIntBetween(0, 10)) .collect(Collectors.toList()); } - - private static ResourceDeprecationChecker createResourceChecker( - String name, - BiFunction>> check - ) { - return new ResourceDeprecationChecker() { - - @Override - public Map> check(ClusterState clusterState, DeprecationInfoAction.Request request) { - return check.apply(clusterState, request); - } - - @Override - public String getName() { - return name; - } - }; - } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java index 2032a6faedc92..475cd3e6a24bc 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IlmPolicyDeprecationCheckerTests.java @@ -90,7 +90,7 @@ public void testLegacyTierSettings() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -136,7 +136,7 @@ public void testFrozenAction() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "ILM policy [deprecated-action] contains the action 'freeze' that is deprecated and will be removed in a future version.", diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 20e1cde4e9b39..5f6928b4cae98 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -30,7 +30,11 @@ import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -43,10 +47,18 @@ public class IndexDeprecationCheckerTests extends ESTestCase { + private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099); + private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); + private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver); + private final TransportDeprecationInfoAction.PrecomputedData emptyPrecomputedData = + new TransportDeprecationInfoAction.PrecomputedData(); private final IndexMetadata.State indexMetdataState; public IndexDeprecationCheckerTests(@Name("indexMetadataState") IndexMetadata.State indexMetdataState) { this.indexMetdataState = indexMetdataState; + emptyPrecomputedData.setOnceNodeSettingsIssues(List.of()); + emptyPrecomputedData.setOncePluginIssues(Map.of()); + emptyPrecomputedData.setOnceTransformConfigs(List.of()); } @ParametersFactory @@ -54,11 +66,6 @@ public static List createParameters() { return List.of(new Object[] { IndexMetadata.State.OPEN }, new Object[] { IndexMetadata.State.CLOSE }); } - private static final IndexVersion OLD_VERSION = IndexVersion.fromId(7170099); - - private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); - private final IndexDeprecationChecker checker = new IndexDeprecationChecker(indexNameExpressionResolver, Map.of()); - public void testOldIndicesCheck() { IndexMetadata indexMetadata = IndexMetadata.builder("test") .settings(settings(OLD_VERSION)) @@ -73,21 +80,22 @@ public void testOldIndicesCheck() { DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, "Old index with a compatibility version < 8.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html", "This index has version: " + OLD_VERSION.toReleaseVersion(), false, singletonMap("reindex_required", true) ); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); List issues = issuesByIndex.get("test"); assertEquals(singletonList(expected), issues); } public void testOldTransformIndicesCheck() { - var checker = new IndexDeprecationChecker(indexNameExpressionResolver, Map.of("test", List.of("test-transform"))); + var checker = new IndexDeprecationChecker(indexNameExpressionResolver); var indexMetadata = indexMetadata("test", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) @@ -95,21 +103,25 @@ public void testOldTransformIndicesCheck() { .build(); var expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 8.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform")) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform"))) + ); assertEquals(singletonList(expected), issuesByIndex.get("test")); } public void testOldIndicesCheckWithMultipleTransforms() { - var checker = new IndexDeprecationChecker( - indexNameExpressionResolver, - Map.of("test", List.of("test-transform1", "test-transform2")) - ); var indexMetadata = indexMetadata("test", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) @@ -117,21 +129,25 @@ public void testOldIndicesCheckWithMultipleTransforms() { .build(); var expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 8.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform1, test-transform2]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform1", "test-transform2")) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform1", "test-transform2"))) + ); assertEquals(singletonList(expected), issuesByIndex.get("test")); } public void testMultipleOldIndicesCheckWithTransforms() { - var checker = new IndexDeprecationChecker( - indexNameExpressionResolver, - Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2")) - ); var indexMetadata1 = indexMetadata("test1", OLD_VERSION); var indexMetadata2 = indexMetadata("test2", OLD_VERSION); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) @@ -143,9 +159,14 @@ public void testMultipleOldIndicesCheckWithTransforms() { List.of( new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 8.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform1]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform1")) ) @@ -154,18 +175,65 @@ public void testMultipleOldIndicesCheckWithTransforms() { List.of( new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < 8.0", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This index has version: " + OLD_VERSION.toReleaseVersion(), + "One or more Transforms write to this index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and requires action before upgrading to 9.0. " + + "The following transforms are configured to write to this index: [test-transform2]. Refer to the " + + "migration guide to learn more about how to prepare transforms destination indices for your upgrade.", false, Map.of("reindex_required", true, "transform_ids", List.of("test-transform2")) ) ) ); - var issuesByIndex = checker.check(clusterState, new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS)); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2"))) + ); assertEquals(expected, issuesByIndex); } + public void testOldIndicesWithIncompatibleDateFormatsCheck() { + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(OLD_VERSION)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "date": { + "type": "date", + "format": "qqqq yyyy" + } + } + }""") + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible date format patterns in old index", + "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23", + "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. " + + "Manual reindexing is required. Field [date] with format pattern [qqqq yyyy].", + false, + null + ); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) { return IndexMetadata.builder(indexName) .settings(settings(indexVersion)) @@ -216,7 +284,8 @@ public void testOldIndicesCheckDataStreamIndex() { .build(); Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -237,14 +306,40 @@ public void testOldIndicesCheckSnapshotIgnored() { Map> issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } public void testOldIndicesIgnoredWarningCheck() { + IndexMetadata indexMetadata = readonlyIndexMetadata("test", OLD_VERSION); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < 8.0 has been ignored", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html", + "This read-only index has version: " + OLD_VERSION.toReleaseVersion() + " and will be supported as read-only in 9.0", + false, + singletonMap("reindex_required", true) + ); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertTrue(issuesByIndex.containsKey("test")); + assertEquals(List.of(expected), issuesByIndex.get("test")); + } + + public void testOldSystemIndicesIgnored() { + // We do not want system indices coming back in the deprecation info API Settings.Builder settings = settings(OLD_VERSION).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); IndexMetadata indexMetadata = IndexMetadata.builder("test") + .system(true) .settings(settings) .numberOfShards(1) .numberOfReplicas(0) @@ -254,20 +349,119 @@ public void testOldIndicesIgnoredWarningCheck() { .metadata(Metadata.builder().put(indexMetadata, true)) .blocks(clusterBlocksForIndices(indexMetadata)) .build(); - DeprecationIssue expected = new DeprecationIssue( + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertThat(issuesByIndex, equalTo(Map.of())); + } + + private IndexMetadata readonlyIndexMetadata(String indexName, IndexVersion indexVersion) { + Settings.Builder settings = settings(indexVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + return IndexMetadata.builder(indexName).settings(settings).numberOfShards(1).numberOfReplicas(0).state(indexMetdataState).build(); + } + + public void testOldTransformIndicesIgnoredCheck() { + var checker = new IndexDeprecationChecker(indexNameExpressionResolver); + var indexMetadata = readonlyIndexMetadata("test", OLD_VERSION); + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + var expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < 8.0 Has Been Ignored", - "https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0", - "This read-only index has version: " + OLD_VERSION.toReleaseVersion() + " and will be supported as read-only in 9.0", + "One or more Transforms write to this old index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", false, - singletonMap("reindex_required", true) + Map.of("reindex_required", true, "transform_ids", List.of("test-transform")) ); - Map> issuesByIndex = checker.check( + var issuesByIndex = checker.check( clusterState, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform"))) ); - assertTrue(issuesByIndex.containsKey("test")); - assertEquals(List.of(expected), issuesByIndex.get("test")); + assertEquals(singletonList(expected), issuesByIndex.get("test")); + } + + public void testOldIndicesIgnoredCheckWithMultipleTransforms() { + var indexMetadata = readonlyIndexMetadata("test", OLD_VERSION); + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + var expected = new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform1, test-transform2]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform1", "test-transform2")) + ); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test", List.of("test-transform1", "test-transform2"))) + ); + assertEquals(singletonList(expected), issuesByIndex.get("test")); + } + + public void testMultipleOldIndicesIgnoredCheckWithTransforms() { + var indexMetadata1 = readonlyIndexMetadata("test1", OLD_VERSION); + var indexMetadata2 = readonlyIndexMetadata("test2", OLD_VERSION); + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata1, true).put(indexMetadata2, true)) + .blocks(clusterBlocksForIndices(indexMetadata1, indexMetadata2)) + .build(); + var expected = Map.of( + "test1", + List.of( + new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform1]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform1")) + ) + ), + "test2", + List.of( + new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < 8.0", + "https://www.elastic.co/guide/en/elastic-stack/9.0/upgrading-elastic-stack.html" + + "#breaking_90_transform_destination_index", + "This index was created in version [" + + OLD_VERSION.toReleaseVersion() + + "] and will be supported as a read-only index in 9.0. " + + "The following transforms are no longer able to write to this index: [test-transform2]. Refer to the " + + "migration guide to learn more about how to handle your transforms destination indices.", + false, + Map.of("reindex_required", true, "transform_ids", List.of("test-transform2")) + ) + ) + ); + var issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + createContextWithTransformConfigs(Map.of("test1", List.of("test-transform1"), "test2", List.of("test-transform2"))) + ); + assertEquals(expected, issuesByIndex); } public void testTranslogRetentionSettings() { @@ -286,7 +480,8 @@ public void testTranslogRetentionSettings() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); List issues = issuesByIndex.get("test"); assertThat( @@ -329,7 +524,8 @@ public void testDefaultTranslogRetentionSettings() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat(issuesByIndex.size(), equalTo(0)); } @@ -349,7 +545,8 @@ public void testIndexDataPathSetting() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); final String expectedUrl = "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-shared-data-path-setting"; @@ -383,7 +580,8 @@ public void testSimpleFSSetting() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -409,7 +607,8 @@ public void testFrozenIndex() { ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(Metadata.builder().put(indexMetadata, true)).build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -452,7 +651,8 @@ public void testCamelCaseDeprecation() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -483,7 +683,8 @@ public void testLegacyTierIndex() { .build(); Map> issuesByIndex = checker.check( state, - new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS) + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData ); assertThat( issuesByIndex.get("test"), @@ -511,4 +712,23 @@ private ClusterBlocks clusterBlocksForIndices(IndexMetadata... indicesMetadatas) } return builder.build(); } + + private TransportDeprecationInfoAction.PrecomputedData createContextWithTransformConfigs(Map> indexToTransform) { + List transforms = new ArrayList<>(); + for (Map.Entry> entry : indexToTransform.entrySet()) { + String index = entry.getKey(); + for (String transform : entry.getValue()) { + transforms.add( + TransformConfig.builder() + .setId(transform) + .setSource(new SourceConfig(randomAlphaOfLength(10))) + .setDest(new DestConfig(index, List.of(), null)) + .build() + ); + } + } + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(transforms); + return precomputedData; + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java new file mode 100644 index 0000000000000..78ddba87b9f85 --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationCheckerTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.deprecation.DeprecationInfoActionResponseTests.createTestDeprecationIssue; +import static org.hamcrest.core.IsEqual.equalTo; + +public class NodeDeprecationCheckerTests extends ESTestCase { + + public void testMergingNodeIssues() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); + mapping.field("enabled", false); + mapping.endObject().endObject(); + + DiscoveryNode node1 = DiscoveryNodeUtils.builder("nodeId1") + .name("node1") + .ephemeralId("ephemeralId1") + .address("hostName1", "hostAddress1", new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .roles(Set.of()) + .build(); + DiscoveryNode node2 = DiscoveryNodeUtils.builder("nodeId2") + .name("node2") + .ephemeralId("ephemeralId2") + .address("hostName2", "hostAddress2", new TransportAddress(TransportAddress.META_ADDRESS, 9500)) + .roles(Set.of()) + .build(); + Map metaMap1 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.1", "setting.2", "setting.3")); + Map metaMap2 = DeprecationIssue.createMetaMapForRemovableSettings(List.of("setting.2", "setting.3")); + DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1); + DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2); + + NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( + new ClusterName(randomAlphaOfLength(5)), + Arrays.asList( + new NodesDeprecationCheckAction.NodeResponse(node1, List.of(foundIssue1)), + new NodesDeprecationCheckAction.NodeResponse(node2, List.of(foundIssue2)) + ), + List.of() + ); + + List result = NodeDeprecationChecker.reduceToDeprecationIssues(nodeDeprecationIssues); + + String details = foundIssue1.getDetails() != null ? foundIssue1.getDetails() + " " : ""; + DeprecationIssue mergedFoundIssue = new DeprecationIssue( + foundIssue1.getLevel(), + foundIssue1.getMessage(), + foundIssue1.getUrl(), + details + "(nodes impacted: [" + node1.getName() + ", " + node2.getName() + "])", + foundIssue1.isResolveDuringRollingUpgrade(), + foundIssue2.getMeta() + ); + assertThat(result, equalTo(List.of(mergedFoundIssue))); + } +} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 0facc29dab589..85c91e4f9a127 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -31,9 +31,11 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.NODE_SETTINGS_CHECKS; +import static org.elasticsearch.xpack.deprecation.NodeDeprecationChecks.SINGLE_NODE_CHECKS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; @@ -155,8 +157,8 @@ public void testSharedDataPathSetting() { .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir()) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); final String expectedUrl = @@ -210,8 +212,8 @@ public void testCheckReservedPrefixedRealmNames() { } final Settings settings = builder.build(); - final List deprecationIssues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + final List deprecationIssues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -237,8 +239,8 @@ public void testCheckReservedPrefixedRealmNames() { public void testSingleDataNodeWatermarkSetting() { Settings settings = Settings.builder().put(DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.getKey(), true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -263,8 +265,8 @@ public void testSingleDataNodeWatermarkSetting() { void monitoringSetting(String settingKey, String value) { Settings settings = Settings.builder().put(settingKey, value).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -287,8 +289,8 @@ void monitoringExporterSetting(String suffix, String value) { String settingKey = "xpack.monitoring.exporters.test." + suffix; Settings settings = Settings.builder().put(settingKey, value).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -312,8 +314,8 @@ void monitoringExporterGroupedSetting(String suffix, String value) { String subSettingKey = settingKey + ".subsetting"; Settings settings = Settings.builder().put(subSettingKey, value).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -338,8 +340,8 @@ void monitoringExporterSecureSetting(String suffix, String value) { secureSettings.setString(settingKey, value); Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, licenseState) ); final String expectedUrl = "https://ela.st/es-deprecation-7-monitoring-settings"; @@ -485,8 +487,8 @@ public void testCheckMonitoringSettingCollectionInterval() { public void testExporterUseIngestPipelineSettings() { Settings settings = Settings.builder().put("xpack.monitoring.exporters.test.use_ingest", true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -511,8 +513,8 @@ public void testExporterPipelineMasterTimeoutSetting() { .put("xpack.monitoring.exporters.test.index.pipeline.master_timeout", TimeValue.timeValueSeconds(10)) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -536,8 +538,8 @@ public void testExporterPipelineMasterTimeoutSetting() { public void testExporterCreateLegacyTemplateSetting() { Settings settings = Settings.builder().put("xpack.monitoring.exporters.test.index.template.create_legacy_templates", true).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -563,8 +565,8 @@ public void testScriptContextCacheSetting() { .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "use-context") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -592,8 +594,8 @@ public void testScriptContextCompilationsRateLimitSetting() { .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "456/7m") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -629,8 +631,8 @@ public void testImplicitScriptContextCacheSetting() { .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "2453") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -667,8 +669,8 @@ public void testScriptContextCacheSizeSetting() { .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), 200) .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -704,8 +706,8 @@ public void testScriptContextCacheExpirationSetting() { .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(contexts.get(1)).getKey(), "2d") .build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -736,8 +738,8 @@ public void testScriptContextCacheExpirationSetting() { public void testEnforceDefaultTierPreferenceSetting() { Settings settings = Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(), randomBoolean()).build(); - List issues = DeprecationChecks.filterChecks( - NODE_SETTINGS_CHECKS, + List issues = filterChecks( + SINGLE_NODE_CHECKS, c -> c.apply(settings, null, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -759,8 +761,8 @@ public void testEnforceDefaultTierPreferenceSetting() { } private List getDeprecationIssues(Settings settings, PluginsAndModules pluginsAndModules) { - final List issues = DeprecationChecks.filterChecks( - DeprecationChecks.NODE_SETTINGS_CHECKS, + final List issues = filterChecks( + NodeDeprecationChecks.SINGLE_NODE_CHECKS, c -> c.apply(settings, pluginsAndModules, ClusterState.EMPTY_STATE, new XPackLicenseState(() -> 0)) ); @@ -827,8 +829,8 @@ public void testDynamicSettings() { } Metadata metadata = metadataBuilder.build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); - final List issues = DeprecationChecks.filterChecks( - DeprecationChecks.NODE_SETTINGS_CHECKS, + final List issues = filterChecks( + NodeDeprecationChecks.SINGLE_NODE_CHECKS, c -> c.apply(nodettings, pluginsAndModules, clusterState, licenseState) ); @@ -860,4 +862,8 @@ public void testCheckNodeAttrData() { ); assertThat(issues, hasItem(expected)); } + + static List filterChecks(List checks, Function mapper) { + return checks.stream().map(mapper).filter(Objects::nonNull).toList(); + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java index 81c0d1c7dc918..4e1b28b341282 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TemplateDeprecationCheckerTests.java @@ -48,12 +48,12 @@ public void testCheckSourceModeInComponentTemplates() throws IOException { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.CRITICAL, + SourceFieldMapper.DEPRECATION_WARNING_TITLE, + "https://ela.st/migrate-source-mode", SourceFieldMapper.DEPRECATION_WARNING, - "https://github.com/elastic/elasticsearch/pull/117172", - null, false, null ); @@ -81,7 +81,7 @@ public void testCheckLegacyTiersInComponentTemplates() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -121,7 +121,7 @@ public void testCheckLegacyTierSettings() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expected = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", @@ -164,7 +164,7 @@ public void testComponentAndComposableTemplateWithSameName() { ) .build(); - Map> issuesByComponentTemplate = checker.check(clusterState, null); + Map> issuesByComponentTemplate = checker.check(clusterState); final DeprecationIssue expectedIndexTemplateIssue = new DeprecationIssue( DeprecationIssue.Level.WARNING, "Configuring tiers via filtered allocation is not recommended.", diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java index 85fa375c09c5f..945068ba3a107 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java @@ -6,22 +6,295 @@ */ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.hamcrest.core.IsEqual; +import org.junit.Assert; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.Response.RESERVED_NAMES; +import static org.elasticsearch.xpack.deprecation.DeprecationInfoActionResponseTests.createTestDeprecationIssue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportDeprecationInfoActionTests extends ESTestCase { + public void testCheckAndCreateResponse() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all"); + mapping.field("enabled", false); + mapping.endObject().endObject(); + + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .putMapping(Strings.toString(mapping)) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + ) + .build(); + + DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + boolean clusterIssueFound = randomBoolean(); + boolean nodeIssueFound = randomBoolean(); + boolean indexIssueFound = randomBoolean(); + boolean dataStreamIssueFound = randomBoolean(); + boolean indexTemplateIssueFound = randomBoolean(); + boolean componentTemplateIssueFound = randomBoolean(); + boolean ilmPolicyIssueFound = randomBoolean(); + DeprecationIssue foundIssue = createTestDeprecationIssue(); + ClusterDeprecationChecker clusterDeprecationChecker = mock(ClusterDeprecationChecker.class); + when(clusterDeprecationChecker.check(any(), any())).thenReturn(clusterIssueFound ? List.of(foundIssue) : List.of()); + List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { + if (indexIssueFound) { + return Map.of("test", List.of(foundIssue)); + } + return Map.of(); + }), createResourceChecker("data_streams", (cs, req) -> { + if (dataStreamIssueFound) { + return Map.of("my-ds", List.of(foundIssue)); + } + return Map.of(); + }), createResourceChecker("templates", (cs, req) -> { + Map> issues = new HashMap<>(); + if (componentTemplateIssueFound) { + issues.put("my-component-template", List.of(foundIssue)); + } + if (indexTemplateIssueFound) { + issues.put("my-index-template", List.of(foundIssue)); + } + return issues; + }), createResourceChecker("ilm_policies", (cs, req) -> { + if (ilmPolicyIssueFound) { + return Map.of("my-policy", List.of(foundIssue)); + } + return Map.of(); + })); + + List nodeDeprecationIssues = nodeIssueFound ? List.of(foundIssue) : List.of(); + + DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(List.of()); + precomputedData.setOncePluginIssues(Map.of()); + precomputedData.setOnceNodeSettingsIssues(nodeDeprecationIssues); + DeprecationInfoAction.Response response = TransportDeprecationInfoAction.checkAndCreateResponse( + state, + resolver, + request, + List.of(), + clusterDeprecationChecker, + resourceCheckers, + precomputedData + ); + + if (clusterIssueFound) { + assertThat(response.getClusterSettingsIssues(), IsEqual.equalTo(List.of(foundIssue))); + } else { + assertThat(response.getClusterSettingsIssues(), empty()); + } + + if (nodeIssueFound) { + assertThat(response.getNodeSettingsIssues(), IsEqual.equalTo(List.of(foundIssue))); + } else { + assertTrue(response.getNodeSettingsIssues().isEmpty()); + } + + if (indexIssueFound) { + assertThat(response.getIndexSettingsIssues(), IsEqual.equalTo(Map.of("test", List.of(foundIssue)))); + } else { + assertTrue(response.getIndexSettingsIssues().isEmpty()); + } + if (dataStreamIssueFound) { + assertThat(response.getDataStreamDeprecationIssues(), IsEqual.equalTo(Map.of("my-ds", List.of(foundIssue)))); + } else { + assertTrue(response.getDataStreamDeprecationIssues().isEmpty()); + } + if (ilmPolicyIssueFound) { + assertThat(response.getIlmPolicyDeprecationIssues(), IsEqual.equalTo(Map.of("my-policy", List.of(foundIssue)))); + } else { + assertTrue(response.getIlmPolicyDeprecationIssues().isEmpty()); + } + if (componentTemplateIssueFound == false && indexTemplateIssueFound == false) { + assertTrue(response.getTemplateDeprecationIssues().isEmpty()); + } else { + if (componentTemplateIssueFound) { + assertThat(response.getTemplateDeprecationIssues().get("my-component-template"), IsEqual.equalTo(List.of(foundIssue))); + } + if (indexTemplateIssueFound) { + assertThat(response.getTemplateDeprecationIssues().get("my-index-template"), IsEqual.equalTo(List.of(foundIssue))); + } + + } + } + + public void testRemoveSkippedSettings() { + Settings.Builder settingsBuilder = settings(IndexVersion.current()); + settingsBuilder.put("some.deprecated.property", "someValue1"); + settingsBuilder.put("some.other.bad.deprecated.property", "someValue2"); + settingsBuilder.put("some.undeprecated.property", "someValue3"); + settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); + Settings inputSettings = settingsBuilder.build(); + IndexMetadata dataStreamIndexMetadata = IndexMetadata.builder("ds-test-index-1") + .settings(inputSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + ComponentTemplate componentTemplate = new ComponentTemplate(Template.builder().settings(inputSettings).build(), null, null); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .template(Template.builder().settings(inputSettings)) + .build(); + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(inputSettings).numberOfShards(1).numberOfReplicas(0)) + .put(dataStreamIndexMetadata, true) + .put(DataStream.builder("ds-test", List.of(dataStreamIndexMetadata.getIndex())).build()) + .indexTemplates( + Map.of( + "my-index-template", + indexTemplate, + "empty-template", + ComposableIndexTemplate.builder().indexPatterns(List.of("random")).build() + ) + ) + .componentTemplates(Map.of("my-component-template", componentTemplate)) + .persistentSettings(inputSettings) + .build(); + + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + AtomicReference visibleClusterSettings = new AtomicReference<>(); + ClusterDeprecationChecker clusterDeprecationChecker = mock(ClusterDeprecationChecker.class); + when(clusterDeprecationChecker.check(any(), any())).thenAnswer(invocationOnMock -> { + ClusterState observedState = invocationOnMock.getArgument(0); + visibleClusterSettings.set(observedState.getMetadata().settings()); + return List.of(); + }); + AtomicReference visibleIndexSettings = new AtomicReference<>(); + AtomicReference visibleComponentTemplateSettings = new AtomicReference<>(); + AtomicReference visibleIndexTemplateSettings = new AtomicReference<>(); + AtomicInteger backingIndicesCount = new AtomicInteger(0); + List resourceCheckers = List.of(createResourceChecker("index_settings", (cs, req) -> { + for (String indexName : resolver.concreteIndexNames(cs, req)) { + visibleIndexSettings.set(cs.metadata().index(indexName).getSettings()); + } + return Map.of(); + }), createResourceChecker("data_streams", (cs, req) -> { + cs.metadata().dataStreams().values().forEach(ds -> backingIndicesCount.set(ds.getIndices().size())); + return Map.of(); + }), createResourceChecker("templates", (cs, req) -> { + cs.metadata() + .componentTemplates() + .values() + .forEach(template -> visibleComponentTemplateSettings.set(template.template().settings())); + cs.metadata().templatesV2().values().forEach(template -> { + if (template.template() != null && template.template().settings() != null) { + visibleIndexTemplateSettings.set(template.template().settings()); + } + }); + return Map.of(); + })); + TransportDeprecationInfoAction.PrecomputedData precomputedData = new TransportDeprecationInfoAction.PrecomputedData(); + precomputedData.setOnceTransformConfigs(List.of()); + precomputedData.setOncePluginIssues(Map.of()); + precomputedData.setOnceNodeSettingsIssues(List.of()); + DeprecationInfoAction.Request request = new DeprecationInfoAction.Request(randomTimeValue(), Strings.EMPTY_ARRAY); + TransportDeprecationInfoAction.checkAndCreateResponse( + state, + resolver, + request, + List.of("some.deprecated.property", "some.other.*.deprecated.property"), + clusterDeprecationChecker, + resourceCheckers, + precomputedData + ); + + settingsBuilder = settings(IndexVersion.current()); + settingsBuilder.put("some.undeprecated.property", "someValue3"); + settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); + + Settings expectedSettings = settingsBuilder.build(); + Settings resultClusterSettings = visibleClusterSettings.get(); + Assert.assertNotNull(resultClusterSettings); + Assert.assertEquals(expectedSettings, visibleClusterSettings.get()); + + Settings resultIndexSettings = visibleIndexSettings.get(); + Assert.assertNotNull(resultIndexSettings); + Assert.assertEquals("someValue3", resultIndexSettings.get("some.undeprecated.property")); + Assert.assertEquals(resultIndexSettings.getAsList("some.undeprecated.list.property"), List.of("someValue4", "someValue5")); + Assert.assertFalse(resultIndexSettings.hasValue("some.deprecated.property")); + Assert.assertFalse(resultIndexSettings.hasValue("some.other.bad.deprecated.property")); + + assertThat(backingIndicesCount.get(), IsEqual.equalTo(1)); + + Assert.assertNotNull(visibleComponentTemplateSettings.get()); + Assert.assertEquals(expectedSettings, visibleComponentTemplateSettings.get()); + Assert.assertNotNull(visibleIndexTemplateSettings.get()); + Assert.assertEquals(expectedSettings, visibleIndexTemplateSettings.get()); + } + + public void testCtorFailure() { + Map> indexNames = Stream.generate(() -> randomAlphaOfLength(10)) + .limit(10) + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + Map> dataStreamNames = Stream.generate(() -> randomAlphaOfLength(10)) + .limit(10) + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + Set shouldCauseFailure = new HashSet<>(RESERVED_NAMES); + for (int i = 0; i < randomIntBetween(1, 100); i++) { + Map> pluginSettingsIssues = randomSubsetOf(3, shouldCauseFailure).stream() + .collect(Collectors.toMap(Function.identity(), (_k) -> List.of())); + expectThrows( + ElasticsearchStatusException.class, + () -> new DeprecationInfoAction.Response( + List.of(), + List.of(), + Map.of("data_streams", dataStreamNames, "index_settings", indexNames), + pluginSettingsIssues + ) + ); + } + } + public void testPluginSettingIssues() { DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null); PlainActionFuture>> future = new PlainActionFuture<>(); @@ -65,6 +338,28 @@ public void testPluginSettingIssuesWithFailures() { assertThat(exception.getCause().getMessage(), containsString("boom")); } + private static ResourceDeprecationChecker createResourceChecker( + String name, + BiFunction>> check + ) { + return new ResourceDeprecationChecker() { + + @Override + public Map> check( + ClusterState clusterState, + DeprecationInfoAction.Request request, + TransportDeprecationInfoAction.PrecomputedData precomputedData + ) { + return check.apply(clusterState, request); + } + + @Override + public String getName() { + return name; + } + }; + } + private static class NamedChecker implements DeprecationChecker { private final String name; diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index 80692efb7474a..a0a37f2bb52d1 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -61,7 +61,7 @@ public void testNodeOperation() { settingsBuilder.put("some.undeprecated.property", "someValue3"); settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); Settings nodeSettings = settingsBuilder.build(); @@ -73,7 +73,10 @@ public void testNodeOperation() { ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); ClusterService clusterService = Mockito.mock(ClusterService.class); when(clusterService.state()).thenReturn(clusterState); - ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, Set.of(DeprecationChecks.SKIP_DEPRECATIONS_SETTING)); + ClusterSettings clusterSettings = new ClusterSettings( + nodeSettings, + Set.of(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING) + ); when((clusterService.getClusterSettings())).thenReturn(clusterSettings); DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); @@ -98,7 +101,7 @@ public void testNodeOperation() { NodesDeprecationCheckAction.NodeRequest nodeRequest = null; AtomicReference visibleNodeSettings = new AtomicReference<>(); AtomicReference visibleClusterStateMetadataSettings = new AtomicReference<>(); - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -109,7 +112,7 @@ public void testNodeOperation() { return null; }; java.util.List< - DeprecationChecks.NodeDeprecationCheck< + NodeDeprecationChecks.NodeDeprecationCheck< Settings, PluginsAndModules, ClusterState, @@ -120,7 +123,7 @@ public void testNodeOperation() { settingsBuilder.put("some.undeprecated.property", "someValue3"); settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); Settings expectedSettings = settingsBuilder.build(); @@ -131,7 +134,7 @@ public void testNodeOperation() { // Testing that the setting is dynamically updatable: Settings newSettings = Settings.builder() - .putList(DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.undeprecated.property")) + .putList(TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.undeprecated.property")) .build(); clusterSettings.applySettings(newSettings); transportNodeDeprecationCheckAction.nodeOperation(nodeRequest, nodeSettingsChecks); @@ -141,7 +144,7 @@ public void testNodeOperation() { settingsBuilder.putList("some.undeprecated.list.property", List.of("someValue4", "someValue5")); // This is the node setting (since this is the node deprecation check), not the cluster setting: settingsBuilder.putList( - DeprecationChecks.SKIP_DEPRECATIONS_SETTING.getKey(), + TransportDeprecationInfoAction.SKIP_DEPRECATIONS_SETTING.getKey(), List.of("some.deprecated.property", "some.other.*.deprecated.property", "some.bad.dynamic.property") ); expectedSettings = settingsBuilder.build(); diff --git a/x-pack/plugin/downsample/qa/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/MixedClusterDownsampleRestIT.java b/x-pack/plugin/downsample/qa/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/MixedClusterDownsampleRestIT.java index a4765271e7300..a05009b6715b0 100644 --- a/x-pack/plugin/downsample/qa/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/MixedClusterDownsampleRestIT.java +++ b/x-pack/plugin/downsample/qa/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/MixedClusterDownsampleRestIT.java @@ -19,13 +19,23 @@ public class MixedClusterDownsampleRestIT extends ESClientYamlSuiteTestCase { @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .withNode(node -> node.version(getOldVersion())) - .withNode(node -> node.version(Version.CURRENT)) - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .build(); + public static ElasticsearchCluster cluster = buildCluster(); + + private static ElasticsearchCluster buildCluster() { + Version oldVersion = getOldVersion(); + var cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .withNode(node -> node.version(getOldVersion())) + .withNode(node -> node.version(Version.CURRENT)) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial"); + + if (oldVersion.before(Version.fromString("8.18.0"))) { + cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper"); + cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService"); + } + return cluster.build(); + } static Version getOldVersion() { return Version.fromString(System.getProperty("tests.old_cluster_version")); diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml new file mode 100644 index 0000000000000..991aa3858d8bc --- /dev/null +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml @@ -0,0 +1,79 @@ +"downsample aggregate field": + - requires: + cluster_features: ["data_stream.downsample.default_aggregate_metric_fix"] + reason: "#119696 fixed" + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [sensor_id] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + sensor_id: + type: keyword + time_series_dimension: true + temperature: + type: aggregate_metric_double + metrics: [min, sum, value_count] + default_metric: sum + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:00:00Z", "sensor_id": "1", "temperature": {"min": 24.7, "sum": 50.2, "value_count": 2}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:30:00Z", "sensor_id": "1", "temperature": {"min": 24.2, "sum": 73.8, "value_count": 3}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:00:00Z", "sensor_id": "1", "temperature": {"min": 25.1, "sum": 51.0, "value_count": 2}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:30:00Z", "sensor_id": "1", "temperature": {"min": 24.8, "sum": 24.8, "value_count": 1}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T20:00:00Z", "sensor_id": "1", "temperature": {"min": 24.6, "sum": 49.1, "value_count": 2}}' + + - do: + indices.put_settings: + index: test + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test + target_index: test-downsample + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: test-downsample + body: + size: 0 + + - match: + hits.total.value: 3 + + - do: + indices.get_mapping: + index: test-downsample + - match: + test-downsample.mappings.properties.temperature: + type: aggregate_metric_double + metrics: [min, sum, value_count] + default_metric: sum + time_series_metric: gauge diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index afa2e95e1284c..917ce781fb1f8 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -119,7 +119,7 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { } catch (Exception e) { throw new AssertionError(e); } - }, 60, TimeUnit.SECONDS); + }, 120, TimeUnit.SECONDS); ensureGreen(targetIndex); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java index 43c68e81e869d..518c044138727 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AbstractDownsampleFieldProducer.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.downsample; +import org.apache.lucene.internal.hppc.IntArrayList; import org.elasticsearch.index.fielddata.FormattedDocValues; import java.io.IOException; @@ -43,5 +44,5 @@ public boolean isEmpty() { return isEmpty; } - public abstract void collect(FormattedDocValues docValues, int docId) throws IOException; + public abstract void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException; } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java index 57137ec429978..eb0d49f728ad1 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java @@ -12,7 +12,7 @@ import java.io.IOException; import java.util.Collection; -public class AggregateMetricFieldSerializer implements DownsampleFieldSerializer { +final class AggregateMetricFieldSerializer implements DownsampleFieldSerializer { private final Collection producers; private final String name; @@ -22,7 +22,7 @@ public class AggregateMetricFieldSerializer implements DownsampleFieldSerializer * @param producers a collection of {@link AbstractDownsampleFieldProducer} instances with the subfields * of the aggregate_metric_double field. */ - public AggregateMetricFieldSerializer(String name, Collection producers) { + AggregateMetricFieldSerializer(String name, Collection producers) { this.name = name; this.producers = producers; } @@ -38,10 +38,22 @@ public void write(XContentBuilder builder) throws IOException { assert name.equals(fieldProducer.name()) : "producer has a different name"; if (fieldProducer.isEmpty() == false) { if (fieldProducer instanceof MetricFieldProducer metricFieldProducer) { - for (MetricFieldProducer.Metric metric : metricFieldProducer.metrics()) { - if (metric.get() != null) { - builder.field(metric.name(), metric.get()); + if (metricFieldProducer instanceof MetricFieldProducer.GaugeMetricFieldProducer gaugeProducer) { + builder.field("max", gaugeProducer.max); + builder.field("min", gaugeProducer.min); + builder.field("sum", gaugeProducer.sum.value()); + builder.field("value_count", gaugeProducer.count); + } else if (metricFieldProducer instanceof MetricFieldProducer.CounterMetricFieldProducer counterProducer) { + builder.field("last_value", counterProducer.lastValue); + } else if (metricFieldProducer instanceof MetricFieldProducer.AggregatedGaugeMetricFieldProducer producer) { + switch (producer.metric) { + case max -> builder.field("max", producer.max); + case min -> builder.field("min", producer.min); + case sum -> builder.field("sum", producer.sum.value()); + case value_count -> builder.field("value_count", producer.count); } + } else { + throw new IllegalStateException(); } } else if (fieldProducer instanceof LabelFieldProducer labelFieldProducer) { LabelFieldProducer.Label label = labelFieldProducer.label(); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java index a451439fadea1..f169b0c672dea 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java @@ -10,18 +10,18 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; public final class AggregateMetricFieldValueFetcher extends FieldValueFetcher { - private final AggregateDoubleMetricFieldType aggMetricFieldType; + private final AggregateMetricDoubleFieldType aggMetricFieldType; private final AbstractDownsampleFieldProducer fieldProducer; AggregateMetricFieldValueFetcher( MappedFieldType fieldType, - AggregateDoubleMetricFieldType aggMetricFieldType, + AggregateMetricDoubleFieldType aggMetricFieldType, IndexFieldData fieldData ) { super(fieldType.name(), fieldType, fieldData); @@ -34,7 +34,7 @@ public AbstractDownsampleFieldProducer fieldProducer() { } private AbstractDownsampleFieldProducer createFieldProducer() { - AggregateDoubleMetricFieldMapper.Metric metric = null; + AggregateMetricDoubleFieldMapper.Metric metric = null; for (var e : aggMetricFieldType.getMetricFields().entrySet()) { NumberFieldMapper.NumberFieldType metricSubField = e.getValue(); if (metricSubField.name().equals(name())) { @@ -47,14 +47,7 @@ private AbstractDownsampleFieldProducer createFieldProducer() { if (aggMetricFieldType.getMetricType() != null) { // If the field is an aggregate_metric_double field, we should use the correct subfields // for each aggregation. This is a downsample-of-downsample case - MetricFieldProducer.Metric metricOperation = switch (metric) { - case max -> new MetricFieldProducer.Max(); - case min -> new MetricFieldProducer.Min(); - case sum -> new MetricFieldProducer.Sum(); - // To compute value_count summary, we must sum all field values - case value_count -> new MetricFieldProducer.Sum(AggregateDoubleMetricFieldMapper.Metric.value_count.name()); - }; - return new MetricFieldProducer.GaugeMetricFieldProducer(aggMetricFieldType.name(), metricOperation); + return new MetricFieldProducer.AggregatedGaugeMetricFieldProducer(aggMetricFieldType.name(), metric); } else { // If field is not a metric, we downsample it as a label return new LabelFieldProducer.AggregateMetricFieldProducer.AggregateMetricFieldProducer(aggMetricFieldType.name(), metric); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java index 69493e6de442e..ba7f9eea8eee5 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.downsample; +import org.apache.lucene.internal.hppc.IntArrayList; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.xcontent.XContentBuilder; @@ -44,16 +45,32 @@ void reset() { isEmpty = true; } - void collect(final Object value) { + void collectOnce(final Object value) { + assert isEmpty; Objects.requireNonNull(value); - if (isEmpty) { - this.value = value; - this.isEmpty = false; - return; - } - if (value.equals(this.value) == false) { - throw new IllegalArgumentException("Dimension value changed without tsid change [" + value + "] != [" + this.value + "]"); + this.value = value; + this.isEmpty = false; + } + + /** + * This is an expensive check, that slows down downsampling significantly. + * Given that index is sorted by tsid as primary key, this shouldn't really happen. + */ + boolean validate(FormattedDocValues docValues, IntArrayList buffer) throws IOException { + for (int i = 0; i < buffer.size(); i++) { + int docId = buffer.get(i); + if (docValues.advanceExact(docId)) { + int docValueCount = docValues.docValueCount(); + for (int j = 0; j < docValueCount; j++) { + var value = docValues.nextValue(); + if (value.equals(this.value) == false) { + assert false : "Dimension value changed without tsid change [" + value + "] != [" + this.value + "]"; + } + } + } } + + return true; } } @@ -68,13 +85,24 @@ public boolean isEmpty() { } @Override - public void collect(FormattedDocValues docValues, int docId) throws IOException { - if (docValues.advanceExact(docId) == false) { + public void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException { + if (dimension.isEmpty == false) { + assert dimension.validate(docValues, docIdBuffer); return; } - int docValueCount = docValues.docValueCount(); - for (int i = 0; i < docValueCount; i++) { - this.dimension.collect(docValues.nextValue()); + + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + int docValueCount = docValues.docValueCount(); + for (int j = 0; j < docValueCount; j++) { + this.dimension.collectOnce(docValues.nextValue()); + } + // Only need to record one dimension value from one document, within in the same tsid-and-time-interval bucket values are the + // same. + return; } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 50149ec2cbe58..51459f8181059 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.internal.hppc.IntArrayList; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -33,6 +34,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocCountFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; @@ -80,6 +82,7 @@ class DownsampleShardIndexer { private static final Logger logger = LogManager.getLogger(DownsampleShardIndexer.class); + private static final int DOCID_BUFFER_SIZE = 8096; public static final int DOWNSAMPLE_BULK_ACTIONS = 10000; public static final ByteSizeValue DOWNSAMPLE_BULK_SIZE = new ByteSizeValue(1, ByteSizeUnit.MB); public static final ByteSizeValue DOWNSAMPLE_MAX_BYTES_IN_FLIGHT = new ByteSizeValue(50, ByteSizeUnit.MB); @@ -338,6 +341,7 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) private class TimeSeriesBucketCollector extends BucketCollector { private final BulkProcessor2 bulkProcessor; private final DownsampleBucketBuilder downsampleBucketBuilder; + private final List leafBucketCollectors = new ArrayList<>(); private long docsProcessed; private long bucketsCreated; long lastTimestamp = Long.MAX_VALUE; @@ -358,90 +362,181 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag docCountProvider.setLeafReaderContext(ctx); // For each field, return a tuple with the downsample field producer and the field value leaf - final AbstractDownsampleFieldProducer[] fieldProducers = new AbstractDownsampleFieldProducer[fieldValueFetchers.size()]; - final FormattedDocValues[] formattedDocValues = new FormattedDocValues[fieldValueFetchers.size()]; - for (int i = 0; i < fieldProducers.length; i++) { - fieldProducers[i] = fieldValueFetchers.get(i).fieldProducer(); - formattedDocValues[i] = fieldValueFetchers.get(i).getLeaf(ctx); + final List nonMetricProducers = new ArrayList<>(); + final List formattedDocValues = new ArrayList<>(); + + final List metricProducers = new ArrayList<>(); + final List numericDocValues = new ArrayList<>(); + for (var fieldValueFetcher : fieldValueFetchers) { + var fieldProducer = fieldValueFetcher.fieldProducer(); + if (fieldProducer instanceof MetricFieldProducer metricFieldProducer) { + metricProducers.add(metricFieldProducer); + numericDocValues.add(fieldValueFetcher.getNumericLeaf(ctx)); + } else { + nonMetricProducers.add(fieldProducer); + formattedDocValues.add(fieldValueFetcher.getLeaf(ctx)); + } } - return new LeafBucketCollector() { - @Override - public void collect(int docId, long owningBucketOrd) throws IOException { - task.addNumReceived(1); - final BytesRef tsidHash = aggCtx.getTsidHash(); - assert tsidHash != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; - final int tsidHashOrd = aggCtx.getTsidHashOrd(); - final long timestamp = timestampField.resolution().roundDownToMillis(aggCtx.getTimestamp()); - - boolean tsidChanged = tsidHashOrd != downsampleBucketBuilder.tsidOrd(); - if (tsidChanged || timestamp < lastHistoTimestamp) { - lastHistoTimestamp = Math.max( - rounding.round(timestamp), - searchExecutionContext.getIndexSettings().getTimestampBounds().startTime() - ); - } - task.setLastSourceTimestamp(timestamp); - task.setLastTargetTimestamp(lastHistoTimestamp); - - if (logger.isTraceEnabled()) { - logger.trace( - "Doc: [{}] - _tsid: [{}], @timestamp: [{}] -> downsample bucket ts: [{}]", - docId, - DocValueFormat.TIME_SERIES_ID.format(tsidHash), - timestampFormat.format(timestamp), - timestampFormat.format(lastHistoTimestamp) - ); - } + var leafBucketCollector = new LeafDownsampleCollector( + aggCtx, + docCountProvider, + nonMetricProducers.toArray(new AbstractDownsampleFieldProducer[0]), + formattedDocValues.toArray(new FormattedDocValues[0]), + metricProducers.toArray(new MetricFieldProducer[0]), + numericDocValues.toArray(new SortedNumericDoubleValues[0]) + ); + leafBucketCollectors.add(leafBucketCollector); + return leafBucketCollector; + } + + void bulkCollection() throws IOException { + // The leaf bucket collectors with newer timestamp go first, to correctly capture the last value for counters and labels. + leafBucketCollectors.sort((o1, o2) -> -Long.compare(o1.firstTimeStampForBulkCollection, o2.firstTimeStampForBulkCollection)); + for (LeafDownsampleCollector leafBucketCollector : leafBucketCollectors) { + leafBucketCollector.leafBulkCollection(); + } + } - /* - * Sanity checks to ensure that we receive documents in the correct order - * - _tsid must be sorted in ascending order - * - @timestamp must be sorted in descending order within the same _tsid - */ - BytesRef lastTsid = downsampleBucketBuilder.tsid(); - assert lastTsid == null || lastTsid.compareTo(tsidHash) <= 0 - : "_tsid is not sorted in ascending order: [" - + DocValueFormat.TIME_SERIES_ID.format(lastTsid) - + "] -> [" - + DocValueFormat.TIME_SERIES_ID.format(tsidHash) - + "]"; - assert tsidHash.equals(lastTsid) == false || lastTimestamp >= timestamp - : "@timestamp is not sorted in descending order: [" - + timestampFormat.format(lastTimestamp) - + "] -> [" - + timestampFormat.format(timestamp) - + "]"; - lastTimestamp = timestamp; - - if (tsidChanged || downsampleBucketBuilder.timestamp() != lastHistoTimestamp) { - // Flush downsample doc if not empty - if (downsampleBucketBuilder.isEmpty() == false) { - XContentBuilder doc = downsampleBucketBuilder.buildDownsampleDocument(); - indexBucket(doc); - } - - // Create new downsample bucket - if (tsidChanged) { - downsampleBucketBuilder.resetTsid(tsidHash, tsidHashOrd, lastHistoTimestamp); - } else { - downsampleBucketBuilder.resetTimestamp(lastHistoTimestamp); - } - bucketsCreated++; + class LeafDownsampleCollector extends LeafBucketCollector { + + final AggregationExecutionContext aggCtx; + final DocCountProvider docCountProvider; + final FormattedDocValues[] formattedDocValues; + final AbstractDownsampleFieldProducer[] nonMetricProducers; + + final MetricFieldProducer[] metricProducers; + final SortedNumericDoubleValues[] numericDocValues; + + // Capture the first timestamp in order to determine which leaf collector's leafBulkCollection() is invoked first. + long firstTimeStampForBulkCollection; + final IntArrayList docIdBuffer = new IntArrayList(DOCID_BUFFER_SIZE); + final long timestampBoundStartTime = searchExecutionContext.getIndexSettings().getTimestampBounds().startTime(); + + LeafDownsampleCollector( + AggregationExecutionContext aggCtx, + DocCountProvider docCountProvider, + AbstractDownsampleFieldProducer[] nonMetricProducers, + FormattedDocValues[] formattedDocValues, + MetricFieldProducer[] metricProducers, + SortedNumericDoubleValues[] numericDocValues + ) { + assert nonMetricProducers.length == formattedDocValues.length; + assert metricProducers.length == numericDocValues.length; + + this.aggCtx = aggCtx; + this.docCountProvider = docCountProvider; + this.nonMetricProducers = nonMetricProducers; + this.formattedDocValues = formattedDocValues; + this.metricProducers = metricProducers; + this.numericDocValues = numericDocValues; + } + + @Override + public void collect(int docId, long owningBucketOrd) throws IOException { + task.addNumReceived(1); + final BytesRef tsidHash = aggCtx.getTsidHash(); + assert tsidHash != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; + final int tsidHashOrd = aggCtx.getTsidHashOrd(); + final long timestamp = timestampField.resolution().roundDownToMillis(aggCtx.getTimestamp()); + + boolean tsidChanged = tsidHashOrd != downsampleBucketBuilder.tsidOrd(); + if (tsidChanged || timestamp < lastHistoTimestamp) { + lastHistoTimestamp = Math.max(rounding.round(timestamp), timestampBoundStartTime); + } + task.setLastSourceTimestamp(timestamp); + task.setLastTargetTimestamp(lastHistoTimestamp); + + if (logger.isTraceEnabled()) { + logger.trace( + "Doc: [{}] - _tsid: [{}], @timestamp: [{}] -> downsample bucket ts: [{}]", + docId, + DocValueFormat.TIME_SERIES_ID.format(tsidHash), + timestampFormat.format(timestamp), + timestampFormat.format(lastHistoTimestamp) + ); + } + + assert assertTsidAndTimestamp(tsidHash, timestamp); + lastTimestamp = timestamp; + + if (tsidChanged || downsampleBucketBuilder.timestamp() != lastHistoTimestamp) { + bulkCollection(); + // Flush downsample doc if not empty + if (downsampleBucketBuilder.isEmpty() == false) { + XContentBuilder doc = downsampleBucketBuilder.buildDownsampleDocument(); + indexBucket(doc); } - final int docCount = docCountProvider.getDocCount(docId); - downsampleBucketBuilder.collectDocCount(docCount); - // Iterate over all field values and collect the doc_values for this docId - for (int i = 0; i < fieldProducers.length; i++) { - AbstractDownsampleFieldProducer fieldProducer = fieldProducers[i]; - FormattedDocValues docValues = formattedDocValues[i]; - fieldProducer.collect(docValues, docId); + // Create new downsample bucket + if (tsidChanged) { + downsampleBucketBuilder.resetTsid(tsidHash, tsidHashOrd, lastHistoTimestamp); + } else { + downsampleBucketBuilder.resetTimestamp(lastHistoTimestamp); } - docsProcessed++; - task.setDocsProcessed(docsProcessed); + bucketsCreated++; } - }; + + if (docIdBuffer.isEmpty()) { + firstTimeStampForBulkCollection = aggCtx.getTimestamp(); + } + // buffer.add() always delegates to system.arraycopy() and checks buffer size for resizing purposes: + docIdBuffer.buffer[docIdBuffer.elementsCount++] = docId; + if (docIdBuffer.size() == DOCID_BUFFER_SIZE) { + bulkCollection(); + } + } + + void leafBulkCollection() throws IOException { + if (docIdBuffer.isEmpty()) { + return; + } + + if (logger.isDebugEnabled()) { + logger.debug("buffered {} docids", docIdBuffer.size()); + } + + downsampleBucketBuilder.collectDocCount(docIdBuffer, docCountProvider); + // Iterate over all field values and collect the doc_values for this docId + for (int i = 0; i < nonMetricProducers.length; i++) { + AbstractDownsampleFieldProducer fieldProducer = nonMetricProducers[i]; + FormattedDocValues docValues = formattedDocValues[i]; + fieldProducer.collect(docValues, docIdBuffer); + } + for (int i = 0; i < metricProducers.length; i++) { + MetricFieldProducer metricFieldProducer = metricProducers[i]; + SortedNumericDoubleValues numericDoubleValues = numericDocValues[i]; + metricFieldProducer.collect(numericDoubleValues, docIdBuffer); + } + + docsProcessed += docIdBuffer.size(); + task.setDocsProcessed(docsProcessed); + + // buffer.clean() also overwrites all slots with zeros + docIdBuffer.elementsCount = 0; + } + + /** + * Sanity checks to ensure that we receive documents in the correct order + * - _tsid must be sorted in ascending order + * - @timestamp must be sorted in descending order within the same _tsid + */ + boolean assertTsidAndTimestamp(BytesRef tsidHash, long timestamp) { + BytesRef lastTsid = downsampleBucketBuilder.tsid(); + assert lastTsid == null || lastTsid.compareTo(tsidHash) <= 0 + : "_tsid is not sorted in ascending order: [" + + DocValueFormat.TIME_SERIES_ID.format(lastTsid) + + "] -> [" + + DocValueFormat.TIME_SERIES_ID.format(tsidHash) + + "]"; + assert tsidHash.equals(lastTsid) == false || lastTimestamp >= timestamp + : "@timestamp is not sorted in descending order: [" + + timestampFormat.format(lastTimestamp) + + "] -> [" + + timestampFormat.format(timestamp) + + "]"; + return true; + } } private void indexBucket(XContentBuilder doc) { @@ -464,6 +559,7 @@ public void preCollection() { @Override public void postCollection() throws IOException { // Flush downsample doc if not empty + bulkCollection(); if (downsampleBucketBuilder.isEmpty() == false) { XContentBuilder doc = downsampleBucketBuilder.buildDownsampleDocument(); indexBucket(doc); @@ -545,8 +641,15 @@ public void resetTimestamp(long timestamp) { } } - public void collectDocCount(int docCount) { - this.docCount += docCount; + public void collectDocCount(IntArrayList buffer, DocCountProvider docCountProvider) throws IOException { + if (docCountProvider.alwaysOne()) { + this.docCount += buffer.size(); + } else { + for (int i = 0; i < buffer.size(); i++) { + int docId = buffer.get(i); + this.docCount += docCountProvider.getDocCount(docId); + } + } } public XContentBuilder buildDownsampleDocument() throws IOException { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java index 3657e4989ccbd..8974fd6dc8ad2 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java @@ -10,12 +10,14 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; import java.util.ArrayList; import java.util.Collections; @@ -50,6 +52,11 @@ public FormattedDocValues getLeaf(LeafReaderContext context) { return fieldData.load(context).getFormattedValues(format); } + public SortedNumericDoubleValues getNumericLeaf(LeafReaderContext context) { + LeafNumericFieldData numericFieldData = (LeafNumericFieldData) fieldData.load(context); + return numericFieldData.getDoubleValues(); + } + public AbstractDownsampleFieldProducer fieldProducer() { return fieldProducer; } @@ -82,7 +89,7 @@ static List create(SearchExecutionContext context, String[] f MappedFieldType fieldType = context.getFieldType(field); assert fieldType != null : "Unknown field type for field: [" + field + "]"; - if (fieldType instanceof AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType aggMetricFieldType) { + if (fieldType instanceof AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType) { // If the field is an aggregate_metric_double field, we should load all its subfields // This is a downsample-of-downsample case for (NumberFieldMapper.NumberFieldType metricSubField : aggMetricFieldType.getMetricFields().values()) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java index b211c5bfb0d12..7e2c5061d0b50 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java @@ -7,12 +7,13 @@ package org.elasticsearch.xpack.downsample; +import org.apache.lucene.internal.hppc.IntArrayList; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.HistogramValue; import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.ArrayList; @@ -114,25 +115,31 @@ public void write(XContentBuilder builder) throws IOException { } @Override - public void collect(FormattedDocValues docValues, int docId) throws IOException { + public void collect(FormattedDocValues docValues, IntArrayList docIdBuffer) throws IOException { if (isEmpty() == false) { return; } - if (docValues.advanceExact(docId) == false) { - return; - } - int docValuesCount = docValues.docValueCount(); - assert docValuesCount > 0; - isEmpty = false; - if (docValuesCount == 1) { - label.collect(docValues.nextValue()); - } else { - Object[] values = new Object[docValuesCount]; - for (int i = 0; i < docValuesCount; i++) { - values[i] = docValues.nextValue(); + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + int docValuesCount = docValues.docValueCount(); + assert docValuesCount > 0; + isEmpty = false; + if (docValuesCount == 1) { + label.collect(docValues.nextValue()); + } else { + var values = new Object[docValuesCount]; + for (int j = 0; j < docValuesCount; j++) { + values[j] = docValues.nextValue(); + } + label.collect(values); } - label.collect(values); + // Only need to record one label value from one document, within in the same tsid-and-time-interval we only keep the first + // with downsampling. + return; } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java index 1305ea8ab38d2..642ca910a111c 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java @@ -7,9 +7,12 @@ package org.elasticsearch.xpack.downsample; +import org.apache.lucene.internal.hppc.IntArrayList; import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; import java.io.IOException; @@ -19,267 +22,177 @@ * gauge and metric types. */ abstract sealed class MetricFieldProducer extends AbstractDownsampleFieldProducer { - /** - * a list of metrics that will be computed for the field - */ - private final Metric[] metrics; - MetricFieldProducer(String name, Metric... metrics) { + MetricFieldProducer(String name) { super(name); - this.metrics = metrics; - } - - /** - * Reset all values collected for the field - */ - public void reset() { - for (Metric metric : metrics) { - metric.reset(); - } - isEmpty = true; - } - - /** return the list of metrics that are computed for the field */ - public Metric[] metrics() { - return metrics; - } - - /** Collect the value of a raw field and compute all downsampled metrics */ - void collect(Number value) { - for (MetricFieldProducer.Metric metric : metrics()) { - metric.collect(value); - } - isEmpty = false; } @Override - public void collect(FormattedDocValues docValues, int docId) throws IOException { - if (docValues.advanceExact(docId) == false) { - return; - } - int docValuesCount = docValues.docValueCount(); - for (int i = 0; i < docValuesCount; i++) { - Number num = (Number) docValues.nextValue(); - collect(num); - } + public void collect(FormattedDocValues docValues, IntArrayList buffer) throws IOException { + assert false : "MetricFieldProducer does not support formatted doc values"; + throw new UnsupportedOperationException(); } - abstract static sealed class Metric { - final String name; - - /** - * Abstract class that defines how a metric is computed. - * @param name the name of the metric as it will be output in the downsampled document - */ - protected Metric(String name) { - this.name = name; - } - - public String name() { - return name; - } - - abstract void collect(Number number); - - abstract Number get(); - - abstract void reset(); - } + public abstract void collect(SortedNumericDoubleValues docValues, IntArrayList buffer) throws IOException; /** - * Metric implementation that computes the maximum of all values of a field + * {@link MetricFieldProducer} implementation for a counter metric field */ - static final class Max extends Metric { - private Double max; + static final class CounterMetricFieldProducer extends MetricFieldProducer { - Max() { - super("max"); - } + static final double NO_VALUE = Double.MIN_VALUE; - @Override - void collect(Number value) { - this.max = max != null ? Math.max(value.doubleValue(), max) : value.doubleValue(); - } + double lastValue = NO_VALUE; - @Override - Number get() { - return max; + CounterMetricFieldProducer(String name) { + super(name); } @Override - void reset() { - max = null; - } - } - - /** - * Metric implementation that computes the minimum of all values of a field - */ - static final class Min extends Metric { - private Double min; - - Min() { - super("min"); - } + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + if (isEmpty() == false) { + return; + } - @Override - void collect(Number value) { - this.min = min != null ? Math.min(value.doubleValue(), min) : value.doubleValue(); + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId)) { + isEmpty = false; + lastValue = docValues.nextValue(); + return; + } + } } @Override - Number get() { - return min; + public void reset() { + isEmpty = true; + lastValue = NO_VALUE; } @Override - void reset() { - min = null; + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(name(), lastValue); + } } } - /** - * Metric implementation that computes the sum of all values of a field - */ - static final class Sum extends Metric { - private final CompensatedSum kahanSummation = new CompensatedSum(); - - Sum() { - super("sum"); - } - - Sum(String name) { - super(name); - } - - @Override - void collect(Number value) { - kahanSummation.add(value.doubleValue()); - } - - @Override - Number get() { - return kahanSummation.value(); - } - - @Override - void reset() { - kahanSummation.reset(0, 0); - } - } + static final double MAX_NO_VALUE = -Double.MAX_VALUE; + static final double MIN_NO_VALUE = Double.MAX_VALUE; /** - * Metric implementation that counts all values collected for a metric field + * {@link MetricFieldProducer} implementation for a gauge metric field */ - static final class ValueCount extends Metric { - private long count; + static final class GaugeMetricFieldProducer extends MetricFieldProducer { - ValueCount() { - super("value_count"); - } + double max = MAX_NO_VALUE; + double min = MIN_NO_VALUE; + final CompensatedSum sum = new CompensatedSum(); + long count; - @Override - void collect(Number value) { - count++; + GaugeMetricFieldProducer(String name) { + super(name); } @Override - Number get() { - return count; + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + int docValuesCount = docValues.docValueCount(); + for (int j = 0; j < docValuesCount; j++) { + double value = docValues.nextValue(); + this.max = Math.max(value, max); + this.min = Math.min(value, min); + sum.add(value); + count++; + } + } } @Override - void reset() { + public void reset() { + isEmpty = true; + max = MAX_NO_VALUE; + min = MIN_NO_VALUE; + sum.reset(0, 0); count = 0; } - } - - /** - * Metric implementation that stores the last value over time for a metric. This implementation - * assumes that field values are collected sorted by descending order by time. In this case, - * it assumes that the last value of the time is the first value collected. Eventually, - * the implementation of this class end up storing the first value it is empty and then - * ignoring everything else. - */ - static final class LastValue extends Metric { - private Number lastValue; - - LastValue() { - super("last_value"); - } @Override - void collect(Number value) { - if (lastValue == null) { - lastValue = value; + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.startObject(name()); + builder.field("min", min); + builder.field("max", max); + builder.field("sum", sum.value()); + builder.field("value_count", count); + builder.endObject(); } } - - @Override - Number get() { - return lastValue; - } - - @Override - void reset() { - lastValue = null; - } } - /** - * {@link MetricFieldProducer} implementation for a counter metric field - */ - static final class CounterMetricFieldProducer extends MetricFieldProducer { + // For downsampling downsampled indices: + static final class AggregatedGaugeMetricFieldProducer extends MetricFieldProducer { - CounterMetricFieldProducer(String name) { - super(name, new LastValue()); - } + final AggregateMetricDoubleFieldMapper.Metric metric; - @Override - public void collect(FormattedDocValues docValues, int docId) throws IOException { - // Counter producers only collect the last_value. Since documents are - // collected by descending timestamp order, the producer should only - // process the first value for every tsid. So, it will only collect the - // field if no value has been set before. - if (isEmpty()) { - super.collect(docValues, docId); - } - } + double max = MAX_NO_VALUE; + double min = MIN_NO_VALUE; + final CompensatedSum sum = new CompensatedSum(); + long count; - public Object value() { - assert metrics().length == 1 : "Single value producers must have only one metric"; - return metrics()[0].get(); + AggregatedGaugeMetricFieldProducer(String name, AggregateMetricDoubleFieldMapper.Metric metric) { + super(name); + this.metric = metric; } @Override - public void write(XContentBuilder builder) throws IOException { - if (isEmpty() == false) { - builder.field(name(), value()); + public void collect(SortedNumericDoubleValues docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + int docValuesCount = docValues.docValueCount(); + for (int j = 0; j < docValuesCount; j++) { + double value = docValues.nextValue(); + switch (metric) { + case min -> min = Math.min(value, min); + case max -> max = Math.max(value, max); + case sum -> sum.add(value); + // This is the reason why we can't use GaugeMetricFieldProducer + // For downsampled indices aggregate metric double's value count field needs to be summed. + // (Note: not using CompensatedSum here should be ok given that value_count is mapped as long) + case value_count -> count += Math.round(value); + } + } } } - } - - /** - * {@link MetricFieldProducer} implementation for a gauge metric field - */ - static final class GaugeMetricFieldProducer extends MetricFieldProducer { - - GaugeMetricFieldProducer(String name) { - this(name, new Min(), new Max(), new Sum(), new ValueCount()); - } - GaugeMetricFieldProducer(String name, Metric... metrics) { - super(name, metrics); + @Override + public void reset() { + isEmpty = true; + max = MAX_NO_VALUE; + min = MIN_NO_VALUE; + sum.reset(0, 0); + count = 0; } @Override public void write(XContentBuilder builder) throws IOException { if (isEmpty() == false) { builder.startObject(name()); - for (MetricFieldProducer.Metric metric : metrics()) { - if (metric.get() != null) { - builder.field(metric.name(), metric.get()); - } + switch (metric) { + case min -> builder.field("min", min); + case max -> builder.field("max", max); + case sum -> builder.field("sum", sum.value()); + case value_count -> builder.field("value_count", count); } builder.endObject(); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 4463ba3c16a5d..a2a69a5b3ec24 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -76,7 +76,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState; import org.elasticsearch.xpack.core.downsample.DownsampleShardTask; @@ -91,6 +91,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -739,6 +740,39 @@ private static void addTimestampField( .endObject(); } + // public for testing + public record AggregateMetricDoubleFieldSupportedMetrics(String defaultMetric, List supportedMetrics) {} + + // public for testing + public static AggregateMetricDoubleFieldSupportedMetrics getSupportedMetrics( + final TimeSeriesParams.MetricType metricType, + final Map fieldProperties + ) { + boolean sourceIsAggregate = fieldProperties.get("type").equals(AggregateMetricDoubleFieldMapper.CONTENT_TYPE); + List supportedAggs = List.of(metricType.supportedAggs()); + + if (sourceIsAggregate) { + @SuppressWarnings("unchecked") + List currentAggs = (List) fieldProperties.get(AggregateMetricDoubleFieldMapper.Names.METRICS); + supportedAggs = supportedAggs.stream().filter(currentAggs::contains).toList(); + } + + assert supportedAggs.size() > 0; + + String defaultMetric = "max"; + if (supportedAggs.contains(defaultMetric) == false) { + defaultMetric = supportedAggs.get(0); + } + if (sourceIsAggregate) { + defaultMetric = Objects.requireNonNullElse( + (String) fieldProperties.get(AggregateMetricDoubleFieldMapper.Names.DEFAULT_METRIC), + defaultMetric + ); + } + + return new AggregateMetricDoubleFieldSupportedMetrics(defaultMetric, supportedAggs); + } + private static void addMetricFieldMapping(final XContentBuilder builder, final String field, final Map fieldProperties) throws IOException { final TimeSeriesParams.MetricType metricType = TimeSeriesParams.MetricType.fromString( @@ -752,12 +786,11 @@ private static void addMetricFieldMapping(final XContentBuilder builder, final S builder.field(fieldProperty, fieldProperties.get(fieldProperty)); } } else { - final String[] supportedAggsArray = metricType.supportedAggs(); - // We choose max as the default metric - final String defaultMetric = List.of(supportedAggsArray).contains("max") ? "max" : supportedAggsArray[0]; - builder.field("type", AggregateDoubleMetricFieldMapper.CONTENT_TYPE) - .array(AggregateDoubleMetricFieldMapper.Names.METRICS, supportedAggsArray) - .field(AggregateDoubleMetricFieldMapper.Names.DEFAULT_METRIC, defaultMetric) + var supported = getSupportedMetrics(metricType, fieldProperties); + + builder.field("type", AggregateMetricDoubleFieldMapper.CONTENT_TYPE) + .stringListField(AggregateMetricDoubleFieldMapper.Names.METRICS, supported.supportedMetrics) + .field(AggregateMetricDoubleFieldMapper.Names.DEFAULT_METRIC, supported.defaultMetric) .field(TIME_SERIES_METRIC_PARAM, metricType); } builder.endObject(); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 33a436a1c7f1b..70c9d6753f691 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1117,25 +1117,31 @@ private void bulkIndex(SourceSupplier sourceSupplier) throws IOException { } private void bulkIndex(final String indexName, final SourceSupplier sourceSupplier, int docCount) throws IOException { - BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); - bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - for (int i = 0; i < docCount; i++) { - IndexRequest indexRequest = new IndexRequest(indexName).opType(DocWriteRequest.OpType.CREATE); - XContentBuilder source = sourceSupplier.get(); - indexRequest.source(source); - bulkRequestBuilder.add(indexRequest); - } - BulkResponse bulkResponse = bulkRequestBuilder.get(); + // Index in such a way that we always have multiple segments, so that we test DownsampleShardIndexer in a more realistic scenario: + // (also makes failures more reproducible) int duplicates = 0; - for (BulkItemResponse response : bulkResponse.getItems()) { - if (response.isFailed()) { - if (response.getFailure().getCause() instanceof VersionConflictEngineException) { - // A duplicate event was created by random generator. We should not fail for this - // reason. - logger.debug("We tried to insert a duplicate: [{}]", response.getFailureMessage()); - duplicates++; - } else { - fail("Failed to index data: " + bulkResponse.buildFailureMessage()); + for (int i = 0; i < docCount;) { + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + int max = Math.min(i + 100, docCount); + for (int j = i; j < max; j++) { + IndexRequest indexRequest = new IndexRequest(indexName).opType(DocWriteRequest.OpType.CREATE); + XContentBuilder source = sourceSupplier.get(); + indexRequest.source(source); + bulkRequestBuilder.add(indexRequest); + } + i = max; + BulkResponse bulkResponse = bulkRequestBuilder.get(); + for (BulkItemResponse response : bulkResponse.getItems()) { + if (response.isFailed()) { + if (response.getFailure().getCause() instanceof VersionConflictEngineException) { + // A duplicate event was created by random generator. We should not fail for this + // reason. + logger.debug("We tried to insert a duplicate: [{}]", response.getFailureMessage()); + duplicates++; + } else { + fail("Failed to index data: " + bulkResponse.buildFailureMessage()); + } } } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java index 844eb1b8e27d8..b3b18cdfd17b3 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.downsample; +import org.apache.lucene.internal.hppc.IntArrayList; import org.elasticsearch.common.Strings; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -93,7 +94,7 @@ public Object nextValue() { return "aaaa"; } }; - producer.collect(docValues, 1); + producer.collect(docValues, IntArrayList.from(1)); // producer.collect("dummy", "aaaa"); assertFalse(producer.isEmpty()); assertEquals("aaaa", producer.label().get()); @@ -129,7 +130,7 @@ public Object nextValue() { } }; - producer.collect(docValues, 1); + producer.collect(docValues, IntArrayList.from(1)); assertFalse(producer.isEmpty()); assertEquals("a\0value_a", (((Object[]) producer.label().get())[0]).toString()); assertEquals("b\0value_b", (((Object[]) producer.label().get())[1]).toString()); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java index 34ccbc5c7202d..a5cb121181577 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java @@ -7,7 +7,10 @@ package org.elasticsearch.xpack.downsample; +import org.apache.lucene.internal.hppc.IntArrayList; +import org.apache.lucene.internal.hppc.IntDoubleHashMap; import org.elasticsearch.common.Strings; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -16,114 +19,129 @@ public class MetricFieldProducerTests extends AggregatorTestCase { - public void testMinCountMetric() { - MetricFieldProducer.Metric metric = new MetricFieldProducer.Min(); - assertNull(metric.get()); - metric.collect(40); - metric.collect(5.5); - metric.collect(12.2); - metric.collect(55); - assertEquals(5.5, metric.get()); - metric.reset(); - assertNull(metric.get()); + public void testMinCountMetric() throws IOException { + var instance = new MetricFieldProducer.GaugeMetricFieldProducer(randomAlphaOfLength(10)); + assertEquals(Double.MAX_VALUE, instance.min, 0); + var docIdBuffer = IntArrayList.from(0, 1, 2, 3); + var values = createValuesInstance(docIdBuffer, 40, 5.5, 12.2, 55); + instance.collect(values, docIdBuffer); + assertEquals(5.5, instance.min, 0); + instance.reset(); + assertEquals(Double.MAX_VALUE, instance.min, 0); } - public void testMaxCountMetric() { - MetricFieldProducer.Metric metric = new MetricFieldProducer.Max(); - assertNull(metric.get()); - metric.collect(5.5); - metric.collect(12.2); - metric.collect(55); - assertEquals(55d, metric.get()); - metric.reset(); - assertNull(metric.get()); + public void testMaxCountMetric() throws IOException { + var instance = new MetricFieldProducer.GaugeMetricFieldProducer(randomAlphaOfLength(10)); + assertEquals(-Double.MAX_VALUE, instance.max, 0); + var docIdBuffer = IntArrayList.from(0, 1, 2); + var values = createValuesInstance(docIdBuffer, 5.5, 12.2, 55); + instance.collect(values, docIdBuffer); + assertEquals(55d, instance.max, 0); + instance.reset(); + assertEquals(-Double.MAX_VALUE, instance.max, 0); } - public void testSumCountMetric() { - MetricFieldProducer.Metric metric = new MetricFieldProducer.Sum(); - assertEquals(0d, metric.get()); - metric.collect(5.5); - metric.collect(12.2); - metric.collect(55); - assertEquals(72.7, metric.get()); - metric.reset(); - assertEquals(0d, metric.get()); + public void testSumCountMetric() throws IOException { + var instance = new MetricFieldProducer.GaugeMetricFieldProducer(randomAlphaOfLength(10)); + assertEquals(0, instance.sum.value(), 0); + var docIdBuffer = IntArrayList.from(0, 1, 2); + var values = createValuesInstance(docIdBuffer, 5.5, 12.2, 55); + instance.collect(values, docIdBuffer); + assertEquals(72.7, instance.sum.value(), 0); + instance.reset(); + assertEquals(0, instance.sum.value(), 0); } /** * Testing summation accuracy. * Tests stolen from SumAggregatorTests#testSummationAccuracy */ - public void testSummationAccuracy() { - MetricFieldProducer.Metric metric = new MetricFieldProducer.Sum(); + public void testSummationAccuracy() throws IOException { + var instance = new MetricFieldProducer.GaugeMetricFieldProducer(randomAlphaOfLength(10)); + assertEquals(0, instance.sum.value(), 0); + var docIdBuffer = IntArrayList.from(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // Summing up a normal array and expect an accurate value - double[] values = new double[] { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7 }; - for (int i = 0; i < values.length; i++) { - metric.collect(values[i]); - } - assertEquals(15.3, metric.get().doubleValue(), Double.MIN_NORMAL); + var values = createValuesInstance(docIdBuffer, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7); + instance.collect(values, docIdBuffer); + assertEquals(15.3, instance.sum.value(), Double.MIN_NORMAL); // Summing up an array which contains NaN and infinities and expect a result same as naive summation - metric.reset(); + instance.reset(); int n = randomIntBetween(5, 10); + docIdBuffer = new IntArrayList(n); + double[] valueArray = new double[n]; double sum = 0; for (int i = 0; i < n; i++) { + docIdBuffer.add(i); double d = frequently() ? randomFrom(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY) : randomDoubleBetween(Double.MIN_VALUE, Double.MAX_VALUE, true); + valueArray[i] = d; sum += d; - metric.collect(d); } - assertEquals(sum, metric.get().doubleValue(), 1e-10); + values = createValuesInstance(docIdBuffer, valueArray); + instance.collect(values, docIdBuffer); + assertEquals(sum, instance.sum.value(), 1e-10); // Summing up some big double values and expect infinity result - metric.reset(); + instance.reset(); n = randomIntBetween(5, 10); + docIdBuffer = new IntArrayList(n); + valueArray = new double[n]; for (int i = 0; i < n; i++) { - metric.collect(Double.MAX_VALUE); + docIdBuffer.add(i); + valueArray[i] = Double.MAX_VALUE; } - assertEquals(Double.POSITIVE_INFINITY, metric.get().doubleValue(), 0d); + values = createValuesInstance(docIdBuffer, valueArray); + instance.collect(values, docIdBuffer); + assertEquals(Double.POSITIVE_INFINITY, instance.sum.value(), 0d); - metric.reset(); + instance.reset(); + n = randomIntBetween(5, 10); + docIdBuffer = new IntArrayList(n); + valueArray = new double[n]; for (int i = 0; i < n; i++) { - metric.collect(-Double.MAX_VALUE); + docIdBuffer.add(i); + valueArray[i] = -Double.MAX_VALUE; } - assertEquals(Double.NEGATIVE_INFINITY, metric.get().doubleValue(), 0d); + values = createValuesInstance(docIdBuffer, valueArray); + instance.collect(values, docIdBuffer); + assertEquals(Double.NEGATIVE_INFINITY, instance.sum.value(), 0d); } - public void testValueCountMetric() { - MetricFieldProducer.Metric metric = new MetricFieldProducer.ValueCount(); - assertEquals(0L, metric.get()); - metric.collect(40); - metric.collect(30); - metric.collect(20); - assertEquals(3L, metric.get()); - metric.reset(); - assertEquals(0L, metric.get()); + public void testValueCountMetric() throws IOException { + var instance = new MetricFieldProducer.GaugeMetricFieldProducer(randomAlphaOfLength(10)); + assertEquals(0, instance.count); + var docIdBuffer = IntArrayList.from(0, 1, 2); + var values = createValuesInstance(docIdBuffer, 40, 30, 20); + instance.collect(values, docIdBuffer); + assertEquals(3L, instance.count); + instance.reset(); + assertEquals(0, instance.count); } - public void testLastValueMetric() { - MetricFieldProducer.Metric metric = new MetricFieldProducer.LastValue(); - assertNull(metric.get()); - metric.collect(40); - metric.collect(30); - metric.collect(20); - assertEquals(40, metric.get()); - metric.reset(); - assertNull(metric.get()); + public void testLastValueMetric() throws IOException { + var instance = new MetricFieldProducer.CounterMetricFieldProducer(randomAlphaOfLength(10)); + assertEquals(Double.MIN_VALUE, instance.lastValue, 0); + var docIdBuffer = IntArrayList.from(0, 1, 2); + var values = createValuesInstance(docIdBuffer, 40, 30, 20); + instance.collect(values, docIdBuffer); + assertEquals(40, instance.lastValue, 0); + instance.reset(); + assertEquals(Double.MIN_VALUE, instance.lastValue, 0); } public void testCounterMetricFieldProducer() throws IOException { final String field = "field"; var producer = new MetricFieldProducer.CounterMetricFieldProducer(field); assertTrue(producer.isEmpty()); - producer.collect(55.0); - producer.collect(12.2); - producer.collect(5.5); + var docIdBuffer = IntArrayList.from(0, 1, 2); + var valuesInstance = createValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); + + producer.collect(valuesInstance, docIdBuffer); assertFalse(producer.isEmpty()); - Object o = producer.value(); - assertEquals(55.0, o); + assertEquals(55.0, producer.lastValue, 0); assertEquals("field", producer.name()); XContentBuilder builder = JsonXContent.contentBuilder().startObject(); @@ -136,9 +154,9 @@ public void testGaugeMetricFieldProducer() throws IOException { final String field = "field"; MetricFieldProducer producer = new MetricFieldProducer.GaugeMetricFieldProducer(field); assertTrue(producer.isEmpty()); - producer.collect(55.0); - producer.collect(12.2); - producer.collect(5.5); + var docIdBuffer = IntArrayList.from(0, 1, 2); + var valuesInstance = createValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); + producer.collect(valuesInstance, docIdBuffer); assertFalse(producer.isEmpty()); @@ -149,4 +167,29 @@ public void testGaugeMetricFieldProducer() throws IOException { assertEquals(field, producer.name()); } + + static SortedNumericDoubleValues createValuesInstance(IntArrayList docIdBuffer, double... values) { + return new SortedNumericDoubleValues() { + + final IntDoubleHashMap docIdToValue = IntDoubleHashMap.from(docIdBuffer.toArray(), values); + + int currentDocId = -1; + + @Override + public boolean advanceExact(int target) throws IOException { + currentDocId = target; + return docIdToValue.containsKey(target); + } + + @Override + public double nextValue() throws IOException { + return docIdToValue.get(currentDocId); + } + + @Override + public int docValueCount() { + return 1; + } + }; + } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index fb699fd7c3418..1b2cc32e12a65 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -13,12 +13,16 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import java.util.List; +import java.util.Map; import java.util.UUID; +import static org.hamcrest.Matchers.is; + public class TransportDownsampleActionTests extends ESTestCase { public void testCopyIndexMetadata() { // GIVEN @@ -107,4 +111,25 @@ private static void assertTargetSettings(final IndexMetadata indexMetadata, fina settings.get(IndexMetadata.SETTING_CREATION_DATE) ); } + + public void testGetSupportedMetrics() { + TimeSeriesParams.MetricType metricType = TimeSeriesParams.MetricType.GAUGE; + Map fieldProperties = Map.of( + "type", + "aggregate_metric_double", + "metrics", + List.of("max", "sum"), + "default_metric", + "sum" + ); + + var supported = TransportDownsampleAction.getSupportedMetrics(metricType, fieldProperties); + assertThat(supported.defaultMetric(), is("sum")); + assertThat(supported.supportedMetrics(), is(List.of("max", "sum"))); + + fieldProperties = Map.of("type", "integer"); + supported = TransportDownsampleAction.getSupportedMetrics(metricType, fieldProperties); + assertThat(supported.defaultMetric(), is("max")); + assertThat(supported.supportedMetrics(), is(List.of(metricType.supportedAggs()))); + } } diff --git a/x-pack/plugin/ent-search/build.gradle b/x-pack/plugin/ent-search/build.gradle index 52634ad788d97..9da7d7e95716c 100644 --- a/x-pack/plugin/ent-search/build.gradle +++ b/x-pack/plugin/ent-search/build.gradle @@ -1,6 +1,8 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { name = 'x-pack-ent-search' @@ -35,25 +37,54 @@ dependencies { javaRestTestImplementation(project(path: xpackModule('core'))) javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation(project(':modules:lang-mustache')) + + yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) } -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.autoconfiguration.enabled', 'false' - user username: 'x_pack_rest_user', password: 'x-pack-test-password' +restResources { + restApi { + include '_common', + 'bulk', + 'cluster', + 'connector', + 'nodes', + 'indices', + 'index', + 'query_rules', + 'search_application', + 'xpack', + 'security', + 'search', + 'ml' + } } -tasks.named("dependencyLicenses").configure { +tasks.named("dependencyLicenses") { mapping from: /jackson.*/, to: 'jackson' } -tasks.named("thirdPartyAudit").configure { +tasks.named("thirdPartyAudit") { ignoreMissingClasses( // [missing classes] SLF4j includes an optional class that depends on an extension class (!) 'org.slf4j.ext.EventData' ) } +tasks.named("yamlRestTest") { + usesDefaultDistribution("uses the xpack/usage api") +} + +tasks.named("yamlRestTestV7CompatTest") { + usesDefaultDistribution("uses the xpack/usage api") +} + +tasks.named("yamlRestTestV7CompatTransform") { task -> + // Behavioral Analytics is deprecated with 9.0.0. + task.addAllowedWarning("Behavioral Analytics is deprecated and will be removed in a future release.") +} + addQaCheckDependencies(project) +artifacts { + restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) +} diff --git a/x-pack/plugin/ent-search/qa/rest/build.gradle b/x-pack/plugin/ent-search/qa/rest/build.gradle deleted file mode 100644 index 5b04a326f142c..0000000000000 --- a/x-pack/plugin/ent-search/qa/rest/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' - -dependencies { - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -restResources { - restApi { - include '_common', - 'bulk', - 'cluster', - 'connector', - 'nodes', - 'indices', - 'index', - 'query_rules', - 'search_application', - 'xpack', - 'security', - 'search', - 'ml' - } -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - extraConfigFile 'roles.yml', file('roles.yml') - user username: 'entsearch-superuser', password: 'entsearch-superuser-password', role: 'superuser' - user username: 'entsearch-admin', password: 'entsearch-admin-password', role: 'admin' - user username: 'entsearch-user', password: 'entsearch-user-password', role: 'user' - user username: 'entsearch-unprivileged', password: 'entsearch-unprivileged-password', role: 'unprivileged' -} - -artifacts { - restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) -} diff --git a/x-pack/plugin/ent-search/src/javaRestTest/java/org/elasticsearch/xpack/entsearch/ConnectorSecretsSystemIndexIT.java b/x-pack/plugin/ent-search/src/javaRestTest/java/org/elasticsearch/xpack/entsearch/ConnectorSecretsSystemIndexIT.java index 730ad1d83a318..c847f58c02267 100644 --- a/x-pack/plugin/ent-search/src/javaRestTest/java/org/elasticsearch/xpack/entsearch/ConnectorSecretsSystemIndexIT.java +++ b/x-pack/plugin/ent-search/src/javaRestTest/java/org/elasticsearch/xpack/entsearch/ConnectorSecretsSystemIndexIT.java @@ -16,10 +16,12 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; import java.io.IOException; import java.util.Map; @@ -28,6 +30,14 @@ public class ConnectorSecretsSystemIndexIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("x-pack-ent-search") + .setting("xpack.security.enabled", "true") + .setting("xpack.security.autoconfiguration.enabled", "false") + .user("x_pack_rest_user", "x-pack-test-password") + .build(); + static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( "x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING @@ -91,4 +101,9 @@ private String getPostSecretJson() throws IOException { private Map getResponseMap(Response response) throws IOException { return XContentHelper.convertToMap(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity()), false); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java index 3e573929731fb..7f107660ba1de 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java @@ -252,8 +252,15 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) { for (MultiGetItemResponse item : multiGetResponse) { String rulesetId = item.getId(); + // this usually happens when the system index does not exist because no query rules were created yet + if (item.isFailed()) { + listener.onFailure(item.getFailure().getFailure()); + return; + } + GetResponse getResponse = item.getResponse(); + // this happens when an individual query ruleset cannot be found if (getResponse.isExists() == false) { listener.onFailure(new ResourceNotFoundException("query ruleset " + rulesetId + " not found")); return; diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java b/x-pack/plugin/ent-search/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java similarity index 59% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java rename to x-pack/plugin/ent-search/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java index 5f74ea9c5abc8..d7b706d2ad6e6 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java +++ b/x-pack/plugin/ent-search/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java @@ -12,11 +12,28 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class EnterpriseSearchRestIT extends ESClientYamlSuiteTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "true") + .setting("xpack.security.autoconfiguration.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .rolesFile(Resource.fromClasspath("roles.yml")) + .user("entsearch-superuser", "entsearch-superuser-password", "superuser", false) + .user("entsearch-admin", "entsearch-admin-password", "admin", false) + .user("entsearch-user", "entsearch-user-password", "user", false) + .user("entsearch-unprivileged", "entsearch-unprivileged-password", "unprivileged", false) + .build(); + public EnterpriseSearchRestIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -37,4 +54,9 @@ protected Settings restClientSettings() { final String value = basicAuthHeaderValue("entsearch-admin", new SecureString("entsearch-admin-password".toCharArray())); return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", value).build(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/10_basic.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/10_basic.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/10_basic.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/10_basic.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/100_connector_update_error.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/100_connector_update_error.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/100_connector_update_error.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/100_connector_update_error.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/110_connector_update_name.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/110_connector_update_name.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/110_connector_update_name.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/110_connector_update_name.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/120_connector_update_service_type.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/120_connector_update_service_type.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/120_connector_update_service_type.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/120_connector_update_service_type.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/150_connector_update_status.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/150_connector_update_status.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/150_connector_update_status.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/150_connector_update_status.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/160_connector_update_api_key_id.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/160_connector_update_api_key_id.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/160_connector_update_api_key_id.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/160_connector_update_api_key_id.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/40_connector_update_pipeline.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/40_connector_update_pipeline.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/40_connector_update_pipeline.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/40_connector_update_pipeline.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/50_connector_update_scheduling.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/50_connector_update_scheduling.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/50_connector_update_scheduling.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/50_connector_update_scheduling.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/60_connector_update_filtering.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/60_connector_update_filtering.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/60_connector_update_filtering.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/60_connector_update_filtering.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/70_connector_check_in.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/70_connector_check_in.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/70_connector_check_in.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/70_connector_check_in.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/80_connector_update_last_sync_stats.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/80_connector_update_last_sync_stats.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/80_connector_update_last_sync_stats.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/80_connector_update_last_sync_stats.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/90_connector_update_configuration.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/90_connector_update_configuration.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/90_connector_update_configuration.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/90_connector_update_configuration.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/10_connector_secret_post.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/10_connector_secret_post.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/10_connector_secret_post.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/10_connector_secret_post.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/20_connector_secret_put.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/20_connector_secret_put.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/20_connector_secret_put.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/20_connector_secret_put.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/30_connector_secret_get.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/30_connector_secret_get.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/30_connector_secret_get.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/30_connector_secret_get.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/40_connector_secret_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/40_connector_secret_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/40_connector_secret_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/secret/40_connector_secret_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/20_connector_sync_job_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/20_connector_sync_job_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/20_connector_sync_job_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/20_connector_sync_job_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/30_connector_sync_job_check_in.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/30_connector_sync_job_check_in.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/30_connector_sync_job_check_in.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/30_connector_sync_job_check_in.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/40_connector_sync_job_cancel.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/40_connector_sync_job_cancel.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/40_connector_sync_job_cancel.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/40_connector_sync_job_cancel.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/50_connector_sync_job_get.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/50_connector_sync_job_get.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/50_connector_sync_job_get.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/50_connector_sync_job_get.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/60_connector_sync_job_error.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/60_connector_sync_job_error.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/60_connector_sync_job_error.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/60_connector_sync_job_error.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/70_connector_sync_job_update_stats.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/70_connector_sync_job_update_stats.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/70_connector_sync_job_update_stats.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/70_connector_sync_job_update_stats.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/80_connector_sync_job_list.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/80_connector_sync_job_list.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/80_connector_sync_job_list.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/80_connector_sync_job_list.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/30_query_ruleset_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/30_query_ruleset_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/30_query_ruleset_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/30_query_ruleset_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/5_query_rulesets_before_setup.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/5_query_rulesets_before_setup.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/5_query_rulesets_before_setup.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/5_query_rulesets_before_setup.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/60_query_rule_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/60_query_rule_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/60_query_rule_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/60_query_rule_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/80_query_rules_retriever.yml diff --git a/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/81_query_rules_retriever_no_rulesets.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/81_query_rules_retriever_no_rulesets.yml new file mode 100644 index 0000000000000..a5956ed6fdf32 --- /dev/null +++ b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/81_query_rules_retriever_no_rulesets.yml @@ -0,0 +1,32 @@ +setup: + - do: + bulk: + refresh: true + index: test-index1 + body: + - index: + _id: foo + - { "text": "foo - pinned doc for foo" } + +--- +"query rules retriever when the .query-rules system index is missing": + - skip: + features: [ headers ] + - do: + search: + index: test-index1 + body: + retriever: + rule: + match_criteria: + foo: foo + bar: bar + ruleset_ids: + abc + retriever: + standard: + query: + query_string: + query: bar + explain: true + catch: "missing" diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/15_search_application_before_setup.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/15_search_application_before_setup.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/15_search_application_before_setup.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/15_search_application_before_setup.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/20_search_application_put.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/20_search_application_put.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/20_search_application_put.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/20_search_application_put.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/30_search_application_get.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/30_search_application_get.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/30_search_application_get.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/30_search_application_get.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/40_search_application_delete.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/40_search_application_delete.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/40_search_application_delete.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/40_search_application_delete.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/50_search_application_list.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/50_search_application_list.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/50_search_application_list.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/50_search_application_list.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/55_search_application_search.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/55_search_application_search.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/55_search_application_search.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/55_search_application_search.yml diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/src/yamlRestTest/resources/roles.yml similarity index 100% rename from x-pack/plugin/ent-search/qa/rest/roles.yml rename to x-pack/plugin/ent-search/src/yamlRestTest/resources/roles.yml diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index 0f4cde933a51f..b4b0ec8f4929b 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -32,7 +32,7 @@ dependencies { * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * ****************************************************************/ -if (buildParams.isSnapshotBuild()) { +if (buildParams.snapshotBuild) { addQaCheckDependencies(project) } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java index cbede5871f275..ccaf4ce3a8861 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java @@ -71,6 +71,9 @@ Collection verify(LogicalPlan plan) { // start bottom-up plan.forEachUp(p -> { + if (p.getClass().equals(Join.class)) { + failures.add(fail(p, "JOIN command is not supported")); + } if (p.analyzed()) { return; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java index fac8788db0f95..6ae414e29b613 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java @@ -171,6 +171,7 @@ public void execute(ActionListener listener) { private void tumbleWindow(int currentStage, ActionListener listener) { if (allowPartialSequenceResults == false && shardFailures.isEmpty() == false) { doPayload(listener); + return; } if (currentStage > matcher.firstPositiveStage && matcher.hasCandidates() == false) { if (restartWindowFromTailQuery) { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index bec71a9846562..887132ab729e9 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -368,6 +368,13 @@ public void testJoin() { accept(idxr, "foo where serial_event_id == 0"); } + public void testJoinCommand() { + final IndexResolution idxr = loadIndexResolution("mapping-ip.json"); + + assertEquals("1:1: JOIN command is not supported", error(idxr, "join [any where true] [any where true]")); + assertEquals("1:1: JOIN command is not supported", error(idxr, "join [any where true] [any where true] | tail 3")); + } + public void testMultiField() { final IndexResolution idxr = loadIndexResolution("mapping-multi-field.json"); accept(idxr, "foo where multi_field.raw == \"bar\""); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java index 0f1cfbb85039c..c4affdff01f24 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java @@ -29,7 +29,7 @@ import static org.elasticsearch.core.Tuple.tuple; public class MetadataAttribute extends TypedAttribute { - public static final String TIMESTAMP_FIELD = "@timestamp"; + public static final String TIMESTAMP_FIELD = "@timestamp"; // this is not a true metadata attribute public static final String TSID_FIELD = "_tsid"; public static final String SCORE = "_score"; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java index d2d01857a1f73..2ee600eb21e0b 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; import java.io.IOException; -import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; /** @@ -34,7 +33,7 @@ public NameId() { @Override public int hashCode() { - return Objects.hash(id); + return Long.hashCode(id); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java index feeba39756373..667720536d884 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java @@ -110,7 +110,7 @@ public void forEachPropertyUp(Class typeToken, Consumer rule) protected void forEachProperty(Class typeToken, Consumer rule) { for (Object prop : info().properties()) { // skip children (only properties are interesting) - if (prop != children && children.contains(prop) == false && typeToken.isInstance(prop)) { + if (prop != children && typeToken.isInstance(prop) && children.contains(prop) == false) { rule.accept((E) prop); } } @@ -203,20 +203,21 @@ public T transformUp(Class typeToken, Function protected > T transformChildren(Function traversalOperation) { boolean childrenChanged = false; - // stream() could be used but the code is just as complicated without any advantages - // further more, it would include bring in all the associated stream/collector object creation even though in - // most cases the immediate tree would be quite small (0,1,2 elements) - List transformedChildren = new ArrayList<>(children().size()); + // Avoid creating a new array of children if no change is needed. + // And when it happens, look at using replacement to minimize the amount of method invocations. + List transformedChildren = null; - for (T child : children) { + for (int i = 0, s = children.size(); i < s; i++) { + T child = children.get(i); T next = traversalOperation.apply(child); - if (child.equals(next)) { - // use the initial value - next = child; - } else { - childrenChanged = true; + if (child.equals(next) == false) { + // lazy copy + replacement in place + if (childrenChanged == false) { + childrenChanged = true; + transformedChildren = new ArrayList<>(children); + } + transformedChildren.set(i, next); } - transformedChildren.add(next); } return (childrenChanged ? replaceChildrenSameSize(transformedChildren) : (T) this); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/NodeInfo.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/NodeInfo.java index e8ce23bc20fd3..28e4e739085d4 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/NodeInfo.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/NodeInfo.java @@ -52,7 +52,7 @@ final T transform(Function rule, Class typeToken) List children = node.children(); Function realRule = p -> { - if (p != children && false == children.contains(p) && (p == null || typeToken.isInstance(p))) { + if (p != children && (p == null || typeToken.isInstance(p)) && false == children.contains(p)) { return rule.apply(typeToken.cast(p)); } return p; diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 2cdc1bcb7c7a7..2310de66ae86b 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -77,7 +77,7 @@ interface Injected { } tasks.named("test").configure { - if (buildParams.isCi() == false) { + if (buildParams.ci == false) { systemProperty 'generateDocs', true def injected = project.objects.newInstance(Injected) doFirst { @@ -147,7 +147,7 @@ tasks.named("test").configure { * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * ****************************************************************/ -if (buildParams.isSnapshotBuild()) { +if (buildParams.snapshotBuild) { addQaCheckDependencies(project) } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 506ccbce91e1d..46881bf337c89 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -29,6 +29,7 @@ import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.TypeKind; import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; @@ -85,6 +86,7 @@ public class AggregatorImplementer { private final boolean stateTypeHasSeen; private final boolean stateTypeHasFailed; private final boolean valuesIsBytesRef; + private final boolean valuesIsArray; private final List intermediateState; private final List createParameters; @@ -126,7 +128,8 @@ public AggregatorImplementer( elements.getPackageOf(declarationType).toString(), (declarationType.getSimpleName() + "AggregatorFunction").replace("AggregatorAggregator", "Aggregator") ); - this.valuesIsBytesRef = BYTES_REF.equals(TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType())); + this.valuesIsBytesRef = BYTES_REF.equals(valueTypeName()); + this.valuesIsArray = TypeKind.ARRAY.equals(valueTypeKind()); intermediateState = Arrays.stream(interStateAnno).map(IntermediateStateDesc::newIntermediateStateDesc).toList(); } @@ -143,10 +146,11 @@ private TypeName choseStateType() { if (false == initReturn.isPrimitive()) { return initReturn; } + String simpleName = firstUpper(initReturn.toString()); if (warnExceptions.isEmpty()) { - return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "State"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "State"); } - return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "FallibleState"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "FallibleState"); } static String valueType(ExecutableElement init, ExecutableElement combine) { @@ -177,7 +181,7 @@ static ClassName valueBlockType(ExecutableElement init, ExecutableElement combin case "double" -> DOUBLE_BLOCK; case "float" -> FLOAT_BLOCK; case "long" -> LONG_BLOCK; - case "int" -> INT_BLOCK; + case "int", "int[]" -> INT_BLOCK; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_BLOCK; default -> throw new IllegalArgumentException("unknown block type for " + valueType(init, combine)); }; @@ -189,7 +193,7 @@ static ClassName valueVectorType(ExecutableElement init, ExecutableElement combi case "double" -> DOUBLE_VECTOR; case "float" -> FLOAT_VECTOR; case "long" -> LONG_VECTOR; - case "int" -> INT_VECTOR; + case "int", "int[]" -> INT_VECTOR; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_VECTOR; default -> throw new IllegalArgumentException("unknown vector type for " + valueType(init, combine)); }; @@ -390,6 +394,10 @@ private MethodSpec addRawVector(boolean masked) { if (masked) { builder.addParameter(BOOLEAN_VECTOR, "mask"); } + if (valuesIsArray) { + builder.addComment("This type does not support vectors because all values are multi-valued"); + return builder.build(); + } if (stateTypeHasSeen) { builder.addStatement("state.seen(true)"); @@ -437,9 +445,18 @@ private MethodSpec addRawBlock(boolean masked) { } builder.addStatement("int start = block.getFirstValueIndex(p)"); builder.addStatement("int end = start + block.getValueCount(p)"); - builder.beginControlFlow("for (int i = start; i < end; i++)"); - combineRawInput(builder, "block"); - builder.endControlFlow(); + if (valuesIsArray) { + String arrayType = valueTypeString(); + builder.addStatement("$L[] valuesArray = new $L[end - start]", arrayType, arrayType); + builder.beginControlFlow("for (int i = start; i < end; i++)"); + builder.addStatement("valuesArray[i-start] = $L.get$L(i)", "block", firstUpper(arrayType)); + builder.endControlFlow(); + combineRawInputForArray(builder, "valuesArray"); + } else { + builder.beginControlFlow("for (int i = start; i < end; i++)"); + combineRawInput(builder, "block"); + builder.endControlFlow(); + } } builder.endControlFlow(); if (combineValueCount != null) { @@ -450,26 +467,17 @@ private MethodSpec addRawBlock(boolean masked) { private void combineRawInput(MethodSpec.Builder builder, String blockVariable) { TypeName returnType = TypeName.get(combine.getReturnType()); - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } - if (valuesIsBytesRef) { - combineRawInputForBytesRef(builder, blockVariable); - } else if (returnType.isPrimitive()) { - combineRawInputForPrimitive(returnType, builder, blockVariable); - } else if (returnType == TypeName.VOID) { - combineRawInputForVoid(builder, blockVariable); - } else { - throw new IllegalArgumentException("combine must return void or a primitive"); - } - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.failed(true)"); - builder.addStatement("return"); - builder.endControlFlow(); - } + warningsBlock(builder, () -> { + if (valuesIsBytesRef) { + combineRawInputForBytesRef(builder, blockVariable); + } else if (returnType.isPrimitive()) { + combineRawInputForPrimitive(returnType, builder, blockVariable); + } else if (returnType == TypeName.VOID) { + combineRawInputForVoid(builder, blockVariable); + } else { + throw new IllegalArgumentException("combine must return void or a primitive"); + } + }); } private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) { @@ -483,6 +491,10 @@ private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder ); } + private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { + warningsBlock(builder, () -> builder.addStatement("$T.combine(state, $L)", declarationType, arrayVariable)); + } + private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable) { builder.addStatement( "$T.combine(state, $L.get$L(i))", @@ -497,6 +509,21 @@ private void combineRawInputForBytesRef(MethodSpec.Builder builder, String block builder.addStatement("$T.combine(state, $L.getBytesRef(i, scratch))", declarationType, blockVariable); } + private void warningsBlock(MethodSpec.Builder builder, Runnable block) { + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } + block.run(); + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.failed(true)"); + builder.addStatement("return"); + builder.endControlFlow(); + } + } + private MethodSpec addIntermediateInput() { MethodSpec.Builder builder = MethodSpec.methodBuilder("addIntermediateInput"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC).addParameter(PAGE, "page"); @@ -529,20 +556,12 @@ private MethodSpec addIntermediateInput() { builder.nextControlFlow("else if (seen.getBoolean(0))"); } - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } - var state = intermediateState.get(0); - var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; - builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); - builder.addStatement("state.seen(true)"); - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.failed(true)"); - builder.endControlFlow(); - } + warningsBlock(builder, () -> { + var state = intermediateState.get(0); + var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; + builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); + builder.addStatement("state.seen(true)"); + }); builder.endControlFlow(); } else { throw new IllegalArgumentException("Don't know how to combine intermediate input. Define combineIntermediate"); @@ -693,4 +712,21 @@ public void assignToVariable(MethodSpec.Builder builder, int offset) { } } } + + private TypeMirror valueTypeMirror() { + return combine.getParameters().get(combine.getParameters().size() - 1).asType(); + } + + private TypeName valueTypeName() { + return TypeName.get(valueTypeMirror()); + } + + private TypeKind valueTypeKind() { + return valueTypeMirror().getKind(); + } + + private String valueTypeString() { + String valueTypeString = TypeName.get(valueTypeMirror()).toString(); + return valuesIsArray ? valueTypeString.substring(0, valueTypeString.length() - 2) : valueTypeString; + } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index f05fa02536a7f..8224c73936b90 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -20,7 +20,6 @@ import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -28,10 +27,12 @@ import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.TypeKind; import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; import static java.util.stream.Collectors.joining; +import static org.elasticsearch.compute.gen.AggregatorImplementer.firstUpper; import static org.elasticsearch.compute.gen.AggregatorImplementer.valueBlockType; import static org.elasticsearch.compute.gen.AggregatorImplementer.valueVectorType; import static org.elasticsearch.compute.gen.Methods.findMethod; @@ -74,6 +75,7 @@ public class GroupingAggregatorImplementer { private final ExecutableElement combineIntermediate; private final TypeName stateType; private final boolean valuesIsBytesRef; + private final boolean valuesIsArray; private final List createParameters; private final ClassName implementation; private final List intermediateState; @@ -102,7 +104,8 @@ public GroupingAggregatorImplementer( this.combineStates = findMethod(declarationType, "combineStates"); this.combineIntermediate = findMethod(declarationType, "combineIntermediate"); this.evaluateFinal = findMethod(declarationType, "evaluateFinal"); - this.valuesIsBytesRef = BYTES_REF.equals(TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType())); + this.valuesIsBytesRef = BYTES_REF.equals(valueTypeName()); + this.valuesIsArray = TypeKind.ARRAY.equals(valueTypeKind()); this.createParameters = init.getParameters() .stream() .map(Parameter::from) @@ -133,12 +136,11 @@ private TypeName choseStateType() { if (false == initReturn.isPrimitive()) { return initReturn; } - String head = initReturn.toString().substring(0, 1).toUpperCase(Locale.ROOT); - String tail = initReturn.toString().substring(1); + String simpleName = firstUpper(initReturn.toString()); if (warnExceptions.isEmpty()) { - return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "ArrayState"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "ArrayState"); } - return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "FallibleArrayState"); + return ClassName.get("org.elasticsearch.compute.aggregation", simpleName + "FallibleArrayState"); } public JavaFile sourceFile() { @@ -364,6 +366,10 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { // Add bytes_ref scratch var that will be used for bytes_ref blocks/vectors builder.addStatement("$T scratch = new $T()", BYTES_REF, BYTES_REF); } + if (valuesIsArray && valuesIsBlock == false) { + builder.addComment("This type does not support vectors because all values are multi-valued"); + return builder.build(); + } builder.beginControlFlow("for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++)"); { @@ -391,9 +397,18 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { builder.endControlFlow(); builder.addStatement("int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset)"); builder.addStatement("int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset)"); - builder.beginControlFlow("for (int v = valuesStart; v < valuesEnd; v++)"); - combineRawInput(builder, "values", "v"); - builder.endControlFlow(); + if (valuesIsArray) { + String arrayType = valueTypeString(); + builder.addStatement("$L[] valuesArray = new $L[valuesEnd - valuesStart]", arrayType, arrayType); + builder.beginControlFlow("for (int v = valuesStart; v < valuesEnd; v++)"); + builder.addStatement("valuesArray[v-valuesStart] = $L.get$L(v)", "values", firstUpper(arrayType)); + builder.endControlFlow(); + combineRawInputForArray(builder, "valuesArray"); + } else { + builder.beginControlFlow("for (int v = valuesStart; v < valuesEnd; v++)"); + combineRawInput(builder, "values", "v"); + builder.endControlFlow(); + } } else { combineRawInput(builder, "values", "groupPosition + positionOffset"); } @@ -407,70 +422,52 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { } private void combineRawInput(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { - TypeName valueType = TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType()); - String secondParameterGetter = "get" - + valueType.toString().substring(0, 1).toUpperCase(Locale.ROOT) - + valueType.toString().substring(1); + TypeName valueType = valueTypeName(); TypeName returnType = TypeName.get(combine.getReturnType()); - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } - if (valuesIsBytesRef) { - combineRawInputForBytesRef(builder, blockVariable, offsetVariable); - } else if (includeTimestampVector) { - combineRawInputWithTimestamp(builder, offsetVariable); - } else if (valueType.isPrimitive() == false) { - throw new IllegalArgumentException("second parameter to combine must be a primitive"); - } else if (returnType.isPrimitive()) { - combineRawInputForPrimitive(builder, secondParameterGetter, blockVariable, offsetVariable); - } else if (returnType == TypeName.VOID) { - combineRawInputForVoid(builder, secondParameterGetter, blockVariable, offsetVariable); - } else { - throw new IllegalArgumentException("combine must return void or a primitive"); - } - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.setFailed(groupId)"); - builder.endControlFlow(); - } + warningsBlock(builder, () -> { + if (valuesIsBytesRef) { + combineRawInputForBytesRef(builder, blockVariable, offsetVariable); + } else if (includeTimestampVector) { + combineRawInputWithTimestamp(builder, offsetVariable); + } else if (valueType.isPrimitive() == false) { + throw new IllegalArgumentException("second parameter to combine must be a primitive, array or BytesRef: " + valueType); + } else if (returnType.isPrimitive()) { + combineRawInputForPrimitive(builder, blockVariable, offsetVariable); + } else if (returnType == TypeName.VOID) { + combineRawInputForVoid(builder, blockVariable, offsetVariable); + } else { + throw new IllegalArgumentException("combine must return void or a primitive"); + } + }); } - private void combineRawInputForPrimitive( - MethodSpec.Builder builder, - String secondParameterGetter, - String blockVariable, - String offsetVariable - ) { + private void combineRawInputForPrimitive(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { builder.addStatement( - "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L($L)))", + "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.get$L($L)))", declarationType, blockVariable, - secondParameterGetter, + firstUpper(valueTypeName().toString()), offsetVariable ); } - private void combineRawInputForVoid( - MethodSpec.Builder builder, - String secondParameterGetter, - String blockVariable, - String offsetVariable - ) { + private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { + warningsBlock(builder, () -> builder.addStatement("$T.combine(state, groupId, $L)", declarationType, arrayVariable)); + } + + private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { builder.addStatement( - "$T.combine(state, groupId, $L.$L($L))", + "$T.combine(state, groupId, $L.get$L($L))", declarationType, blockVariable, - secondParameterGetter, + firstUpper(valueTypeName().toString()), offsetVariable ); } private void combineRawInputWithTimestamp(MethodSpec.Builder builder, String offsetVariable) { - TypeName valueType = TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType()); - String blockType = valueType.toString().substring(0, 1).toUpperCase(Locale.ROOT) + valueType.toString().substring(1); + String blockType = firstUpper(valueTypeName().toString()); if (offsetVariable.contains(" + ")) { builder.addStatement("var valuePosition = $L", offsetVariable); offsetVariable = "valuePosition"; @@ -489,6 +486,20 @@ private void combineRawInputForBytesRef(MethodSpec.Builder builder, String block builder.addStatement("$T.combine(state, groupId, $L.getBytesRef($L, scratch))", declarationType, blockVariable, offsetVariable); } + private void warningsBlock(MethodSpec.Builder builder, Runnable block) { + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } + block.run(); + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.setFailed(groupId)"); + builder.endControlFlow(); + } + } + private MethodSpec selectedMayContainUnseenGroups() { MethodSpec.Builder builder = MethodSpec.methodBuilder("selectedMayContainUnseenGroups"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC); @@ -544,24 +555,16 @@ private MethodSpec addIntermediateInput() { builder.nextControlFlow("else if (seen.getBoolean(groupPosition + positionOffset))"); } - if (warnExceptions.isEmpty() == false) { - builder.beginControlFlow("try"); - } - var name = intermediateState.get(0).name(); - var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType()); - builder.addStatement( - "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))", - declarationType, - name, - vectorAccessor - ); - if (warnExceptions.isEmpty() == false) { - String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; - builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); - builder.addStatement("warnings.registerException(e)"); - builder.addStatement("state.setFailed(groupId)"); - builder.endControlFlow(); - } + warningsBlock(builder, () -> { + var name = intermediateState.get(0).name(); + var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType()); + builder.addStatement( + "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))", + declarationType, + name, + vectorAccessor + ); + }); builder.endControlFlow(); } else { builder.addStatement("$T.combineIntermediate(state, groupId, " + intermediateStateRowAccess() + ")", declarationType); @@ -657,4 +660,24 @@ private MethodSpec close() { private boolean hasPrimitiveState() { return PRIMITIVE_STATE_PATTERN.matcher(stateType.toString()).matches(); } + + private TypeMirror valueTypeMirror() { + return combine.getParameters().get(combine.getParameters().size() - 1).asType(); + } + + private TypeName valueTypeName() { + return TypeName.get(valueTypeMirror()); + } + + private TypeKind valueTypeKind() { + return valueTypeMirror().getKind(); + } + + private String valueTypeString() { + String valueTypeString = TypeName.get(valueTypeMirror()).toString(); + if (valuesIsArray) { + valueTypeString = valueTypeString.substring(0, valueTypeString.length() - 2); + } + return valueTypeString; + } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java index bd77bd7ff1e46..c20bffff6bd3a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.aggregation; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BytesRefHash; import org.elasticsearch.common.util.LongLongHash; @@ -150,47 +151,128 @@ void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContex blocks[offset] = toBlock(driverContext.blockFactory(), selected); } + /** + * Builds a {@link Block} with the unique values collected for the {@code #selected} + * groups. This is the implementation of the final and intermediate results of the agg. + */ Block toBlock(BlockFactory blockFactory, IntVector selected) { if (values.size() == 0) { return blockFactory.newConstantNullBlock(selected.getPositionCount()); } - BytesRef scratch = new BytesRef(); - try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(selected.getPositionCount())) { + + long selectedCountsSize = 0; + long idsSize = 0; + try { + /* + * Get a count of all groups less than the maximum selected group. Count + * *downwards* so that we can flip the sign on all of the actually selected + * groups. Negative values in this array are always unselected groups. + */ + int selectedCountsLen = selected.max() + 1; + long adjust = RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + selectedCountsLen * Integer.BYTES + ); + blockFactory.adjustBreaker(adjust); + selectedCountsSize = adjust; + int[] selectedCounts = new int[selectedCountsLen]; + for (int id = 0; id < values.size(); id++) { + int group = (int) values.getKey1(id); + if (group < selectedCounts.length) { + selectedCounts[group]--; + } + } + + /* + * Total the selected groups and turn the counts into the start index into a sort-of + * off-by-one running count. It's really the number of values that have been inserted + * into the results before starting on this group. Unselected groups will still + * have negative counts. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will contain 0, 3, -2, 4, 5 + */ + int total = 0; for (int s = 0; s < selected.getPositionCount(); s++) { - int selectedGroup = selected.getInt(s); - /* - * Count can effectively be in three states - 0, 1, many. We use those - * states to buffer the first value, so we can avoid calling - * beginPositionEntry on single valued fields. - */ - int count = 0; - long first = 0; - for (int id = 0; id < values.size(); id++) { - if (values.getKey1(id) == selectedGroup) { - long value = values.getKey2(id); - switch (count) { - case 0 -> first = value; - case 1 -> { - builder.beginPositionEntry(); - builder.appendBytesRef(bytes.get(first, scratch)); - builder.appendBytesRef(bytes.get(value, scratch)); + int group = selected.getInt(s); + int count = -selectedCounts[group]; + selectedCounts[group] = total; + total += count; + } + + /* + * Build a list of ids to insert in order *and* convert the running + * count in selectedCounts[group] into the end index (exclusive) in + * ids for each group. + * Here we use the negative counts to signal that a group hasn't been + * selected and the id containing values for that group is ignored. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will start with 0, 3, -2, 4, 5. + * The counts will end with 3, 4, -2, 5, 9. + */ + adjust = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + total * Integer.BYTES); + blockFactory.adjustBreaker(adjust); + idsSize = adjust; + int[] ids = new int[total]; + for (int id = 0; id < values.size(); id++) { + int group = (int) values.getKey1(id); + if (group < selectedCounts.length && selectedCounts[group] >= 0) { + ids[selectedCounts[group]++] = id; + } + } + + /* + * Insert the ids in order. + */ + BytesRef scratch = new BytesRef(); + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(selected.getPositionCount())) { + int start = 0; + for (int s = 0; s < selected.getPositionCount(); s++) { + int group = selected.getInt(s); + int end = selectedCounts[group]; + int count = end - start; + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> append(builder, ids[start], scratch); + default -> { + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + append(builder, ids[i], scratch); } - default -> builder.appendBytesRef(bytes.get(value, scratch)); + builder.endPositionEntry(); } - count++; } + start = end; } - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> builder.appendBytesRef(bytes.get(first, scratch)); - default -> builder.endPositionEntry(); - } + return builder.build(); } - return builder.build(); + } finally { + blockFactory.adjustBreaker(-selectedCountsSize - idsSize); } } - void enableGroupIdTracking(SeenGroupIds seen) { + private void append(BytesRefBlock.Builder builder, int id, BytesRef scratch) { + BytesRef value = bytes.get(values.getKey2(id), scratch); + builder.appendBytesRef(value); + } + + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java index a8409367bc090..505d3a91991ec 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.util.LongLongHash; @@ -129,46 +130,127 @@ void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContex blocks[offset] = toBlock(driverContext.blockFactory(), selected); } + /** + * Builds a {@link Block} with the unique values collected for the {@code #selected} + * groups. This is the implementation of the final and intermediate results of the agg. + */ Block toBlock(BlockFactory blockFactory, IntVector selected) { if (values.size() == 0) { return blockFactory.newConstantNullBlock(selected.getPositionCount()); } - try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(selected.getPositionCount())) { + + long selectedCountsSize = 0; + long idsSize = 0; + try { + /* + * Get a count of all groups less than the maximum selected group. Count + * *downwards* so that we can flip the sign on all of the actually selected + * groups. Negative values in this array are always unselected groups. + */ + int selectedCountsLen = selected.max() + 1; + long adjust = RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + selectedCountsLen * Integer.BYTES + ); + blockFactory.adjustBreaker(adjust); + selectedCountsSize = adjust; + int[] selectedCounts = new int[selectedCountsLen]; + for (int id = 0; id < values.size(); id++) { + int group = (int) values.getKey1(id); + if (group < selectedCounts.length) { + selectedCounts[group]--; + } + } + + /* + * Total the selected groups and turn the counts into the start index into a sort-of + * off-by-one running count. It's really the number of values that have been inserted + * into the results before starting on this group. Unselected groups will still + * have negative counts. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will contain 0, 3, -2, 4, 5 + */ + int total = 0; for (int s = 0; s < selected.getPositionCount(); s++) { - int selectedGroup = selected.getInt(s); - /* - * Count can effectively be in three states - 0, 1, many. We use those - * states to buffer the first value, so we can avoid calling - * beginPositionEntry on single valued fields. - */ - int count = 0; - double first = 0; - for (int id = 0; id < values.size(); id++) { - if (values.getKey1(id) == selectedGroup) { - double value = Double.longBitsToDouble(values.getKey2(id)); - switch (count) { - case 0 -> first = value; - case 1 -> { - builder.beginPositionEntry(); - builder.appendDouble(first); - builder.appendDouble(value); + int group = selected.getInt(s); + int count = -selectedCounts[group]; + selectedCounts[group] = total; + total += count; + } + + /* + * Build a list of ids to insert in order *and* convert the running + * count in selectedCounts[group] into the end index (exclusive) in + * ids for each group. + * Here we use the negative counts to signal that a group hasn't been + * selected and the id containing values for that group is ignored. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will start with 0, 3, -2, 4, 5. + * The counts will end with 3, 4, -2, 5, 9. + */ + adjust = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + total * Integer.BYTES); + blockFactory.adjustBreaker(adjust); + idsSize = adjust; + int[] ids = new int[total]; + for (int id = 0; id < values.size(); id++) { + int group = (int) values.getKey1(id); + if (group < selectedCounts.length && selectedCounts[group] >= 0) { + ids[selectedCounts[group]++] = id; + } + } + + /* + * Insert the ids in order. + */ + try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(selected.getPositionCount())) { + int start = 0; + for (int s = 0; s < selected.getPositionCount(); s++) { + int group = selected.getInt(s); + int end = selectedCounts[group]; + int count = end - start; + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> append(builder, ids[start]); + default -> { + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + append(builder, ids[i]); } - default -> builder.appendDouble(value); + builder.endPositionEntry(); } - count++; } + start = end; } - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> builder.appendDouble(first); - default -> builder.endPositionEntry(); - } + return builder.build(); } - return builder.build(); + } finally { + blockFactory.adjustBreaker(-selectedCountsSize - idsSize); } } - void enableGroupIdTracking(SeenGroupIds seen) { + private void append(DoubleBlock.Builder builder, int id) { + double value = Double.longBitsToDouble(values.getKey2(id)); + builder.appendDouble(value); + } + + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java index f9e5e1b7b283a..9c50552110183 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.compute.ann.Aggregator; @@ -134,48 +135,130 @@ void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContex blocks[offset] = toBlock(driverContext.blockFactory(), selected); } + /** + * Builds a {@link Block} with the unique values collected for the {@code #selected} + * groups. This is the implementation of the final and intermediate results of the agg. + */ Block toBlock(BlockFactory blockFactory, IntVector selected) { if (values.size() == 0) { return blockFactory.newConstantNullBlock(selected.getPositionCount()); } - try (FloatBlock.Builder builder = blockFactory.newFloatBlockBuilder(selected.getPositionCount())) { + + long selectedCountsSize = 0; + long idsSize = 0; + try { + /* + * Get a count of all groups less than the maximum selected group. Count + * *downwards* so that we can flip the sign on all of the actually selected + * groups. Negative values in this array are always unselected groups. + */ + int selectedCountsLen = selected.max() + 1; + long adjust = RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + selectedCountsLen * Integer.BYTES + ); + blockFactory.adjustBreaker(adjust); + selectedCountsSize = adjust; + int[] selectedCounts = new int[selectedCountsLen]; + for (int id = 0; id < values.size(); id++) { + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group < selectedCounts.length) { + selectedCounts[group]--; + } + } + + /* + * Total the selected groups and turn the counts into the start index into a sort-of + * off-by-one running count. It's really the number of values that have been inserted + * into the results before starting on this group. Unselected groups will still + * have negative counts. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will contain 0, 3, -2, 4, 5 + */ + int total = 0; for (int s = 0; s < selected.getPositionCount(); s++) { - int selectedGroup = selected.getInt(s); - /* - * Count can effectively be in three states - 0, 1, many. We use those - * states to buffer the first value, so we can avoid calling - * beginPositionEntry on single valued fields. - */ - int count = 0; - float first = 0; - for (int id = 0; id < values.size(); id++) { - long both = values.get(id); - int group = (int) (both >>> Float.SIZE); - if (group == selectedGroup) { - float value = Float.intBitsToFloat((int) both); - switch (count) { - case 0 -> first = value; - case 1 -> { - builder.beginPositionEntry(); - builder.appendFloat(first); - builder.appendFloat(value); + int group = selected.getInt(s); + int count = -selectedCounts[group]; + selectedCounts[group] = total; + total += count; + } + + /* + * Build a list of ids to insert in order *and* convert the running + * count in selectedCounts[group] into the end index (exclusive) in + * ids for each group. + * Here we use the negative counts to signal that a group hasn't been + * selected and the id containing values for that group is ignored. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will start with 0, 3, -2, 4, 5. + * The counts will end with 3, 4, -2, 5, 9. + */ + adjust = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + total * Integer.BYTES); + blockFactory.adjustBreaker(adjust); + idsSize = adjust; + int[] ids = new int[total]; + for (int id = 0; id < values.size(); id++) { + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group < selectedCounts.length && selectedCounts[group] >= 0) { + ids[selectedCounts[group]++] = id; + } + } + + /* + * Insert the ids in order. + */ + try (FloatBlock.Builder builder = blockFactory.newFloatBlockBuilder(selected.getPositionCount())) { + int start = 0; + for (int s = 0; s < selected.getPositionCount(); s++) { + int group = selected.getInt(s); + int end = selectedCounts[group]; + int count = end - start; + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> append(builder, ids[start]); + default -> { + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + append(builder, ids[i]); } - default -> builder.appendFloat(value); + builder.endPositionEntry(); } - count++; } + start = end; } - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> builder.appendFloat(first); - default -> builder.endPositionEntry(); - } + return builder.build(); } - return builder.build(); + } finally { + blockFactory.adjustBreaker(-selectedCountsSize - idsSize); } } - void enableGroupIdTracking(SeenGroupIds seen) { + private void append(FloatBlock.Builder builder, int id) { + long both = values.get(id); + float value = Float.intBitsToFloat((int) both); + builder.appendFloat(value); + } + + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java index 2420dcee70712..1e0ca72b8d1a6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesIntAggregator.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.compute.ann.Aggregator; @@ -134,48 +135,130 @@ void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContex blocks[offset] = toBlock(driverContext.blockFactory(), selected); } + /** + * Builds a {@link Block} with the unique values collected for the {@code #selected} + * groups. This is the implementation of the final and intermediate results of the agg. + */ Block toBlock(BlockFactory blockFactory, IntVector selected) { if (values.size() == 0) { return blockFactory.newConstantNullBlock(selected.getPositionCount()); } - try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(selected.getPositionCount())) { + + long selectedCountsSize = 0; + long idsSize = 0; + try { + /* + * Get a count of all groups less than the maximum selected group. Count + * *downwards* so that we can flip the sign on all of the actually selected + * groups. Negative values in this array are always unselected groups. + */ + int selectedCountsLen = selected.max() + 1; + long adjust = RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + selectedCountsLen * Integer.BYTES + ); + blockFactory.adjustBreaker(adjust); + selectedCountsSize = adjust; + int[] selectedCounts = new int[selectedCountsLen]; + for (int id = 0; id < values.size(); id++) { + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group < selectedCounts.length) { + selectedCounts[group]--; + } + } + + /* + * Total the selected groups and turn the counts into the start index into a sort-of + * off-by-one running count. It's really the number of values that have been inserted + * into the results before starting on this group. Unselected groups will still + * have negative counts. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will contain 0, 3, -2, 4, 5 + */ + int total = 0; for (int s = 0; s < selected.getPositionCount(); s++) { - int selectedGroup = selected.getInt(s); - /* - * Count can effectively be in three states - 0, 1, many. We use those - * states to buffer the first value, so we can avoid calling - * beginPositionEntry on single valued fields. - */ - int count = 0; - int first = 0; - for (int id = 0; id < values.size(); id++) { - long both = values.get(id); - int group = (int) (both >>> Integer.SIZE); - if (group == selectedGroup) { - int value = (int) both; - switch (count) { - case 0 -> first = value; - case 1 -> { - builder.beginPositionEntry(); - builder.appendInt(first); - builder.appendInt(value); + int group = selected.getInt(s); + int count = -selectedCounts[group]; + selectedCounts[group] = total; + total += count; + } + + /* + * Build a list of ids to insert in order *and* convert the running + * count in selectedCounts[group] into the end index (exclusive) in + * ids for each group. + * Here we use the negative counts to signal that a group hasn't been + * selected and the id containing values for that group is ignored. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will start with 0, 3, -2, 4, 5. + * The counts will end with 3, 4, -2, 5, 9. + */ + adjust = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + total * Integer.BYTES); + blockFactory.adjustBreaker(adjust); + idsSize = adjust; + int[] ids = new int[total]; + for (int id = 0; id < values.size(); id++) { + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group < selectedCounts.length && selectedCounts[group] >= 0) { + ids[selectedCounts[group]++] = id; + } + } + + /* + * Insert the ids in order. + */ + try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(selected.getPositionCount())) { + int start = 0; + for (int s = 0; s < selected.getPositionCount(); s++) { + int group = selected.getInt(s); + int end = selectedCounts[group]; + int count = end - start; + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> append(builder, ids[start]); + default -> { + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + append(builder, ids[i]); } - default -> builder.appendInt(value); + builder.endPositionEntry(); } - count++; } + start = end; } - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> builder.appendInt(first); - default -> builder.endPositionEntry(); - } + return builder.build(); } - return builder.build(); + } finally { + blockFactory.adjustBreaker(-selectedCountsSize - idsSize); } } - void enableGroupIdTracking(SeenGroupIds seen) { + private void append(IntBlock.Builder builder, int id) { + long both = values.get(id); + int value = (int) both; + builder.appendInt(value); + } + + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java index 4938b8f15edb0..ba04f928b9fb9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesLongAggregator.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.util.LongLongHash; @@ -129,46 +130,127 @@ void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContex blocks[offset] = toBlock(driverContext.blockFactory(), selected); } + /** + * Builds a {@link Block} with the unique values collected for the {@code #selected} + * groups. This is the implementation of the final and intermediate results of the agg. + */ Block toBlock(BlockFactory blockFactory, IntVector selected) { if (values.size() == 0) { return blockFactory.newConstantNullBlock(selected.getPositionCount()); } - try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(selected.getPositionCount())) { + + long selectedCountsSize = 0; + long idsSize = 0; + try { + /* + * Get a count of all groups less than the maximum selected group. Count + * *downwards* so that we can flip the sign on all of the actually selected + * groups. Negative values in this array are always unselected groups. + */ + int selectedCountsLen = selected.max() + 1; + long adjust = RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + selectedCountsLen * Integer.BYTES + ); + blockFactory.adjustBreaker(adjust); + selectedCountsSize = adjust; + int[] selectedCounts = new int[selectedCountsLen]; + for (int id = 0; id < values.size(); id++) { + int group = (int) values.getKey1(id); + if (group < selectedCounts.length) { + selectedCounts[group]--; + } + } + + /* + * Total the selected groups and turn the counts into the start index into a sort-of + * off-by-one running count. It's really the number of values that have been inserted + * into the results before starting on this group. Unselected groups will still + * have negative counts. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will contain 0, 3, -2, 4, 5 + */ + int total = 0; for (int s = 0; s < selected.getPositionCount(); s++) { - int selectedGroup = selected.getInt(s); - /* - * Count can effectively be in three states - 0, 1, many. We use those - * states to buffer the first value, so we can avoid calling - * beginPositionEntry on single valued fields. - */ - int count = 0; - long first = 0; - for (int id = 0; id < values.size(); id++) { - if (values.getKey1(id) == selectedGroup) { - long value = values.getKey2(id); - switch (count) { - case 0 -> first = value; - case 1 -> { - builder.beginPositionEntry(); - builder.appendLong(first); - builder.appendLong(value); + int group = selected.getInt(s); + int count = -selectedCounts[group]; + selectedCounts[group] = total; + total += count; + } + + /* + * Build a list of ids to insert in order *and* convert the running + * count in selectedCounts[group] into the end index (exclusive) in + * ids for each group. + * Here we use the negative counts to signal that a group hasn't been + * selected and the id containing values for that group is ignored. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will start with 0, 3, -2, 4, 5. + * The counts will end with 3, 4, -2, 5, 9. + */ + adjust = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + total * Integer.BYTES); + blockFactory.adjustBreaker(adjust); + idsSize = adjust; + int[] ids = new int[total]; + for (int id = 0; id < values.size(); id++) { + int group = (int) values.getKey1(id); + if (group < selectedCounts.length && selectedCounts[group] >= 0) { + ids[selectedCounts[group]++] = id; + } + } + + /* + * Insert the ids in order. + */ + try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(selected.getPositionCount())) { + int start = 0; + for (int s = 0; s < selected.getPositionCount(); s++) { + int group = selected.getInt(s); + int end = selectedCounts[group]; + int count = end - start; + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> append(builder, ids[start]); + default -> { + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + append(builder, ids[i]); } - default -> builder.appendLong(value); + builder.endPositionEntry(); } - count++; } + start = end; } - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> builder.appendLong(first); - default -> builder.endPositionEntry(); - } + return builder.build(); } - return builder.build(); + } finally { + blockFactory.adjustBreaker(-selectedCountsSize - idsSize); } } - void enableGroupIdTracking(SeenGroupIds seen) { + private void append(LongBlock.Builder builder, int id) { + long value = values.getKey2(id); + builder.appendLong(value); + } + + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java index 63318a2189908..ca89e6f999641 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.common.util.DoubleArray; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -101,7 +102,7 @@ public void collect(double value, int bucket) { // Gathering mode long requiredSize = rootIndex + bucketSize; if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); assert 0 <= next && next < bucketSize @@ -257,19 +258,25 @@ private void swap(long lhs, long rhs) { /** * Allocate storage for more buckets and store the "next gather offset" - * for those new buckets. + * for those new buckets. We always grow the storage by whole bucket's + * worth of slots at a time. We never allocate space for partial buckets. */ - private void grow(long minSize) { + private void grow(int bucket) { long oldMax = values.size(); - values = bigArrays.grow(values, minSize); + assert oldMax % bucketSize == 0; + + long newSize = BigArrays.overSize(((long) bucket + 1) * bucketSize, PageCacheRecycler.DOUBLE_PAGE_SIZE, Double.BYTES); + // Round up to the next full bucket. + newSize = (newSize + bucketSize - 1) / bucketSize; + values = bigArrays.resize(values, newSize * bucketSize); // Set the next gather offsets for all newly allocated buckets. - setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + fillGatherOffsets(oldMax); } /** * Maintain the "next gather offsets" for newly allocated buckets. */ - private void setNextGatherOffsets(long startingAt) { + private void fillGatherOffsets(long startingAt) { int nextOffset = getBucketSize() - 1; for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { setNextGatherOffset(bucketRoot, nextOffset); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java index b490fe193c33f..2bf8edd99f48c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -101,7 +102,7 @@ public void collect(float value, int bucket) { // Gathering mode long requiredSize = rootIndex + bucketSize; if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); assert 0 <= next && next < bucketSize @@ -257,19 +258,25 @@ private void swap(long lhs, long rhs) { /** * Allocate storage for more buckets and store the "next gather offset" - * for those new buckets. + * for those new buckets. We always grow the storage by whole bucket's + * worth of slots at a time. We never allocate space for partial buckets. */ - private void grow(long minSize) { + private void grow(int bucket) { long oldMax = values.size(); - values = bigArrays.grow(values, minSize); + assert oldMax % bucketSize == 0; + + long newSize = BigArrays.overSize(((long) bucket + 1) * bucketSize, PageCacheRecycler.FLOAT_PAGE_SIZE, Float.BYTES); + // Round up to the next full bucket. + newSize = (newSize + bucketSize - 1) / bucketSize; + values = bigArrays.resize(values, newSize * bucketSize); // Set the next gather offsets for all newly allocated buckets. - setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + fillGatherOffsets(oldMax); } /** * Maintain the "next gather offsets" for newly allocated buckets. */ - private void setNextGatherOffsets(long startingAt) { + private void fillGatherOffsets(long startingAt) { int nextOffset = getBucketSize() - 1; for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { setNextGatherOffset(bucketRoot, nextOffset); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java index 04a635d75fe52..257dfe2ebb0bd 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.common.util.IntArray; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -101,7 +102,7 @@ public void collect(int value, int bucket) { // Gathering mode long requiredSize = rootIndex + bucketSize; if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); assert 0 <= next && next < bucketSize @@ -257,19 +258,25 @@ private void swap(long lhs, long rhs) { /** * Allocate storage for more buckets and store the "next gather offset" - * for those new buckets. + * for those new buckets. We always grow the storage by whole bucket's + * worth of slots at a time. We never allocate space for partial buckets. */ - private void grow(long minSize) { + private void grow(int bucket) { long oldMax = values.size(); - values = bigArrays.grow(values, minSize); + assert oldMax % bucketSize == 0; + + long newSize = BigArrays.overSize(((long) bucket + 1) * bucketSize, PageCacheRecycler.INT_PAGE_SIZE, Integer.BYTES); + // Round up to the next full bucket. + newSize = (newSize + bucketSize - 1) / bucketSize; + values = bigArrays.resize(values, newSize * bucketSize); // Set the next gather offsets for all newly allocated buckets. - setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + fillGatherOffsets(oldMax); } /** * Maintain the "next gather offsets" for newly allocated buckets. */ - private void setNextGatherOffsets(long startingAt) { + private void fillGatherOffsets(long startingAt) { int nextOffset = getBucketSize() - 1; for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { setNextGatherOffset(bucketRoot, nextOffset); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java index e08c25256944b..c27467ebb60ff 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -101,7 +102,7 @@ public void collect(long value, int bucket) { // Gathering mode long requiredSize = rootIndex + bucketSize; if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); assert 0 <= next && next < bucketSize @@ -257,19 +258,25 @@ private void swap(long lhs, long rhs) { /** * Allocate storage for more buckets and store the "next gather offset" - * for those new buckets. + * for those new buckets. We always grow the storage by whole bucket's + * worth of slots at a time. We never allocate space for partial buckets. */ - private void grow(long minSize) { + private void grow(int bucket) { long oldMax = values.size(); - values = bigArrays.grow(values, minSize); + assert oldMax % bucketSize == 0; + + long newSize = BigArrays.overSize(((long) bucket + 1) * bucketSize, PageCacheRecycler.LONG_PAGE_SIZE, Long.BYTES); + // Round up to the next full bucket. + newSize = (newSize + bucketSize - 1) / bucketSize; + values = bigArrays.resize(values, newSize * bucketSize); // Set the next gather offsets for all newly allocated buckets. - setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + fillGatherOffsets(oldMax); } /** * Maintain the "next gather offsets" for newly allocated buckets. */ - private void setNextGatherOffsets(long startingAt) { + private void fillGatherOffsets(long startingAt) { int nextOffset = getBucketSize() - 1; for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { setNextGatherOffset(bucketRoot, nextOffset); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java new file mode 100644 index 0000000000000..4d1bd972434b1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunction.java @@ -0,0 +1,182 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link SpatialExtentCartesianShapeDocValuesAggregator}. + * This class is generated. Edit {@code AggregatorImplementer} instead. + */ +public final class SpatialExtentCartesianShapeDocValuesAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("minX", ElementType.INT), + new IntermediateStateDesc("maxX", ElementType.INT), + new IntermediateStateDesc("maxY", ElementType.INT), + new IntermediateStateDesc("minY", ElementType.INT) ); + + private final DriverContext driverContext; + + private final SpatialExtentState state; + + private final List channels; + + public SpatialExtentCartesianShapeDocValuesAggregatorFunction(DriverContext driverContext, + List channels, SpatialExtentState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static SpatialExtentCartesianShapeDocValuesAggregatorFunction create( + DriverContext driverContext, List channels) { + return new SpatialExtentCartesianShapeDocValuesAggregatorFunction(driverContext, channels, SpatialExtentCartesianShapeDocValuesAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + return; + } + // Some positions masked away, others kept + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector, mask); + } else { + addRawBlock(block, mask); + } + } + + private void addRawVector(IntVector vector) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawVector(IntVector vector, BooleanVector mask) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawBlock(IntBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + private void addRawBlock(IntBlock block, BooleanVector mask) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block minXUncast = page.getBlock(channels.get(0)); + if (minXUncast.areAllValuesNull()) { + return; + } + IntVector minX = ((IntBlock) minXUncast).asVector(); + assert minX.getPositionCount() == 1; + Block maxXUncast = page.getBlock(channels.get(1)); + if (maxXUncast.areAllValuesNull()) { + return; + } + IntVector maxX = ((IntBlock) maxXUncast).asVector(); + assert maxX.getPositionCount() == 1; + Block maxYUncast = page.getBlock(channels.get(2)); + if (maxYUncast.areAllValuesNull()) { + return; + } + IntVector maxY = ((IntBlock) maxYUncast).asVector(); + assert maxY.getPositionCount() == 1; + Block minYUncast = page.getBlock(channels.get(3)); + if (minYUncast.areAllValuesNull()) { + return; + } + IntVector minY = ((IntBlock) minYUncast).asVector(); + assert minY.getPositionCount() == 1; + SpatialExtentCartesianShapeDocValuesAggregator.combineIntermediate(state, minX.getInt(0), maxX.getInt(0), maxY.getInt(0), minY.getInt(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = SpatialExtentCartesianShapeDocValuesAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..2fa68f5226488 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentCartesianShapeDocValuesAggregator}. + * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. + */ +public final class SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public SpatialExtentCartesianShapeDocValuesAggregatorFunction aggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeDocValuesAggregatorFunction.create(driverContext, channels); + } + + @Override + public SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "spatial_extent_cartesian_shape_doc of valuess"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..48161b3ea4bf3 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction.java @@ -0,0 +1,219 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.aggregation.SeenGroupIds; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentCartesianShapeDocValuesAggregator}. + * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. + */ +public final class SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("minX", ElementType.INT), + new IntermediateStateDesc("maxX", ElementType.INT), + new IntermediateStateDesc("maxY", ElementType.INT), + new IntermediateStateDesc("minY", ElementType.INT) ); + + private final SpatialExtentGroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction(List channels, + SpatialExtentGroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction(channels, SpatialExtentCartesianShapeDocValuesAggregator.initGrouping(), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + IntBlock valuesBlock = page.getBlock(channels.get(0)); + IntVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + + private void addRawInput(int positionOffset, IntVector groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentCartesianShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block minXUncast = page.getBlock(channels.get(0)); + if (minXUncast.areAllValuesNull()) { + return; + } + IntVector minX = ((IntBlock) minXUncast).asVector(); + Block maxXUncast = page.getBlock(channels.get(1)); + if (maxXUncast.areAllValuesNull()) { + return; + } + IntVector maxX = ((IntBlock) maxXUncast).asVector(); + Block maxYUncast = page.getBlock(channels.get(2)); + if (maxYUncast.areAllValuesNull()) { + return; + } + IntVector maxY = ((IntBlock) maxYUncast).asVector(); + Block minYUncast = page.getBlock(channels.get(3)); + if (minYUncast.areAllValuesNull()) { + return; + } + IntVector minY = ((IntBlock) minYUncast).asVector(); + assert minX.getPositionCount() == maxX.getPositionCount() && minX.getPositionCount() == maxY.getPositionCount() && minX.getPositionCount() == minY.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + SpatialExtentCartesianShapeDocValuesAggregator.combineIntermediate(state, groupId, minX.getInt(groupPosition + positionOffset), maxX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + SpatialExtentGroupingState inState = ((SpatialExtentCartesianShapeDocValuesGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + SpatialExtentCartesianShapeDocValuesAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = SpatialExtentCartesianShapeDocValuesAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java similarity index 81% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java index 86c2df88845f7..62440eba29355 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunction.java @@ -23,10 +23,10 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunction} implementation for {@link SpatialExtentCartesianShapeAggregator}. + * {@link AggregatorFunction} implementation for {@link SpatialExtentCartesianShapeSourceValuesAggregator}. * This class is generated. Edit {@code AggregatorImplementer} instead. */ -public final class SpatialExtentCartesianShapeAggregatorFunction implements AggregatorFunction { +public final class SpatialExtentCartesianShapeSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("minX", ElementType.INT), new IntermediateStateDesc("maxX", ElementType.INT), @@ -39,16 +39,16 @@ public final class SpatialExtentCartesianShapeAggregatorFunction implements Aggr private final List channels; - public SpatialExtentCartesianShapeAggregatorFunction(DriverContext driverContext, + public SpatialExtentCartesianShapeSourceValuesAggregatorFunction(DriverContext driverContext, List channels, SpatialExtentState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; } - public static SpatialExtentCartesianShapeAggregatorFunction create(DriverContext driverContext, - List channels) { - return new SpatialExtentCartesianShapeAggregatorFunction(driverContext, channels, SpatialExtentCartesianShapeAggregator.initSingle()); + public static SpatialExtentCartesianShapeSourceValuesAggregatorFunction create( + DriverContext driverContext, List channels) { + return new SpatialExtentCartesianShapeSourceValuesAggregatorFunction(driverContext, channels, SpatialExtentCartesianShapeSourceValuesAggregator.initSingle()); } public static List intermediateStateDesc() { @@ -90,7 +90,7 @@ public void addRawInput(Page page, BooleanVector mask) { private void addRawVector(BytesRefVector vector) { BytesRef scratch = new BytesRef(); for (int i = 0; i < vector.getPositionCount(); i++) { - SpatialExtentCartesianShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -100,7 +100,7 @@ private void addRawVector(BytesRefVector vector, BooleanVector mask) { if (mask.getBoolean(i) == false) { continue; } - SpatialExtentCartesianShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -113,7 +113,7 @@ private void addRawBlock(BytesRefBlock block) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentCartesianShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -130,7 +130,7 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentCartesianShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -163,7 +163,7 @@ public void addIntermediateInput(Page page) { } IntVector minY = ((IntBlock) minYUncast).asVector(); assert minY.getPositionCount() == 1; - SpatialExtentCartesianShapeAggregator.combineIntermediate(state, minX.getInt(0), maxX.getInt(0), maxY.getInt(0), minY.getInt(0)); + SpatialExtentCartesianShapeSourceValuesAggregator.combineIntermediate(state, minX.getInt(0), maxX.getInt(0), maxY.getInt(0), minY.getInt(0)); } @Override @@ -173,7 +173,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, DriverContext drive @Override public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { - blocks[offset] = SpatialExtentCartesianShapeAggregator.evaluateFinal(state, driverContext); + blocks[offset] = SpatialExtentCartesianShapeSourceValuesAggregator.evaluateFinal(state, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..822a10fbe4794 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentCartesianShapeSourceValuesAggregator}. + * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. + */ +public final class SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public SpatialExtentCartesianShapeSourceValuesAggregatorFunction aggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeSourceValuesAggregatorFunction.create(driverContext, channels); + } + + @Override + public SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "spatial_extent_cartesian_shape_source of valuess"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java similarity index 82% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeGroupingAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java index e14ae0cbcab34..77893dd350b86 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction.java @@ -23,10 +23,10 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentCartesianShapeAggregator}. + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentCartesianShapeSourceValuesAggregator}. * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. */ -public final class SpatialExtentCartesianShapeGroupingAggregatorFunction implements GroupingAggregatorFunction { +public final class SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("minX", ElementType.INT), new IntermediateStateDesc("maxX", ElementType.INT), @@ -39,16 +39,16 @@ public final class SpatialExtentCartesianShapeGroupingAggregatorFunction impleme private final DriverContext driverContext; - public SpatialExtentCartesianShapeGroupingAggregatorFunction(List channels, + public SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction(List channels, SpatialExtentGroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; } - public static SpatialExtentCartesianShapeGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new SpatialExtentCartesianShapeGroupingAggregatorFunction(channels, SpatialExtentCartesianShapeAggregator.initGrouping(), driverContext); + public static SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction(channels, SpatialExtentCartesianShapeSourceValuesAggregator.initGrouping(), driverContext); } public static List intermediateStateDesc() { @@ -112,7 +112,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -121,7 +121,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefVector va BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } @@ -141,7 +141,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -157,7 +157,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val int groupEnd = groupStart + groups.getValueCount(groupPosition); for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); - SpatialExtentCartesianShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentCartesianShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } } @@ -194,7 +194,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page assert minX.getPositionCount() == maxX.getPositionCount() && minX.getPositionCount() == maxY.getPositionCount() && minX.getPositionCount() == minY.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentCartesianShapeAggregator.combineIntermediate(state, groupId, minX.getInt(groupPosition + positionOffset), maxX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentCartesianShapeSourceValuesAggregator.combineIntermediate(state, groupId, minX.getInt(groupPosition + positionOffset), maxX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); } } @@ -203,9 +203,9 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu if (input.getClass() != getClass()) { throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); } - SpatialExtentGroupingState inState = ((SpatialExtentCartesianShapeGroupingAggregatorFunction) input).state; + SpatialExtentGroupingState inState = ((SpatialExtentCartesianShapeSourceValuesGroupingAggregatorFunction) input).state; state.enableGroupIdTracking(new SeenGroupIds.Empty()); - SpatialExtentCartesianShapeAggregator.combineStates(state, groupId, inState, position); + SpatialExtentCartesianShapeSourceValuesAggregator.combineStates(state, groupId, inState, position); } @Override @@ -216,7 +216,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) @Override public void evaluateFinal(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - blocks[offset] = SpatialExtentCartesianShapeAggregator.evaluateFinal(state, selected, driverContext); + blocks[offset] = SpatialExtentCartesianShapeSourceValuesAggregator.evaluateFinal(state, selected, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java index 99b6bff4a0184..49b9ca1bad69d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregatorFunction.java @@ -27,12 +27,12 @@ */ public final class SpatialExtentGeoPointDocValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -136,43 +136,43 @@ private void addRawBlock(LongBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java index c4e456c172879..235bd10c3e8e2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesGroupingAggregatorFunction.java @@ -27,12 +27,12 @@ */ public final class SpatialExtentGeoPointDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -168,40 +168,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoPointDocValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java index b48ef23d103b9..7d31bea612321 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregatorFunction.java @@ -28,12 +28,12 @@ */ public final class SpatialExtentGeoPointSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -141,43 +141,43 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java index 33b11696a098b..d1c715d5b5f35 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction.java @@ -28,12 +28,12 @@ */ public final class SpatialExtentGeoPointSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -173,40 +173,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoPointSourceValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java new file mode 100644 index 0000000000000..d181ae1305c7e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunction.java @@ -0,0 +1,196 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.AggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link SpatialExtentGeoShapeDocValuesAggregator}. + * This class is generated. Edit {@code AggregatorImplementer} instead. + */ +public final class SpatialExtentGeoShapeDocValuesAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); + + private final DriverContext driverContext; + + private final SpatialExtentStateWrappedLongitudeState state; + + private final List channels; + + public SpatialExtentGeoShapeDocValuesAggregatorFunction(DriverContext driverContext, + List channels, SpatialExtentStateWrappedLongitudeState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static SpatialExtentGeoShapeDocValuesAggregatorFunction create(DriverContext driverContext, + List channels) { + return new SpatialExtentGeoShapeDocValuesAggregatorFunction(driverContext, channels, SpatialExtentGeoShapeDocValuesAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + return; + } + // Some positions masked away, others kept + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector, mask); + } else { + addRawBlock(block, mask); + } + } + + private void addRawVector(IntVector vector) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawVector(IntVector vector, BooleanVector mask) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawBlock(IntBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + private void addRawBlock(IntBlock block, BooleanVector mask) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + int[] valuesArray = new int[end - start]; + for (int i = start; i < end; i++) { + valuesArray[i-start] = block.getInt(i); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, valuesArray); + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { + return; + } + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { + return; + } + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { + return; + } + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { + return; + } + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { + return; + } + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { + return; + } + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = SpatialExtentGeoShapeDocValuesAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java similarity index 56% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunctionSupplier.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java index 027adbc01948e..cd36ee8fd14a2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier.java @@ -12,29 +12,29 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentGeoShapeAggregator}. + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentGeoShapeDocValuesAggregator}. * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ -public final class SpatialExtentGeoShapeAggregatorFunctionSupplier implements AggregatorFunctionSupplier { +public final class SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { private final List channels; - public SpatialExtentGeoShapeAggregatorFunctionSupplier(List channels) { + public SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier(List channels) { this.channels = channels; } @Override - public SpatialExtentGeoShapeAggregatorFunction aggregator(DriverContext driverContext) { - return SpatialExtentGeoShapeAggregatorFunction.create(driverContext, channels); + public SpatialExtentGeoShapeDocValuesAggregatorFunction aggregator(DriverContext driverContext) { + return SpatialExtentGeoShapeDocValuesAggregatorFunction.create(driverContext, channels); } @Override - public SpatialExtentGeoShapeGroupingAggregatorFunction groupingAggregator( + public SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction groupingAggregator( DriverContext driverContext) { - return SpatialExtentGeoShapeGroupingAggregatorFunction.create(channels, driverContext); + return SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.create(channels, driverContext); } @Override public String describe() { - return "spatial_extent_geo of shapes"; + return "spatial_extent_geo_shape_doc of valuess"; } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..00df4fe3282e6 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction.java @@ -0,0 +1,231 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation.spatial; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; +import org.elasticsearch.compute.aggregation.IntermediateStateDesc; +import org.elasticsearch.compute.aggregation.SeenGroupIds; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentGeoShapeDocValuesAggregator}. + * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. + */ +public final class SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); + + private final SpatialExtentGroupingStateWrappedLongitudeState state; + + private final List channels; + + private final DriverContext driverContext; + + public SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction(List channels, + SpatialExtentGroupingStateWrappedLongitudeState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction(channels, SpatialExtentGeoShapeDocValuesAggregator.initGrouping(), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + IntBlock valuesBlock = page.getBlock(channels.get(0)); + IntVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + + private void addRawInput(int positionOffset, IntVector groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + int[] valuesArray = new int[valuesEnd - valuesStart]; + for (int v = valuesStart; v < valuesEnd; v++) { + valuesArray[v-valuesStart] = values.getInt(v); + } + SpatialExtentGeoShapeDocValuesAggregator.combine(state, groupId, valuesArray); + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, IntVector values) { + // This type does not support vectors because all values are multi-valued + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { + return; + } + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { + return; + } + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { + return; + } + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { + return; + } + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { + return; + } + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { + return; + } + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + SpatialExtentGeoShapeDocValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + SpatialExtentGroupingStateWrappedLongitudeState inState = ((SpatialExtentGeoShapeDocValuesGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + SpatialExtentGeoShapeDocValuesAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = SpatialExtentGeoShapeDocValuesAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java similarity index 62% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java index 3e651f4eb7921..b72a8f23eb5ca 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunction.java @@ -23,17 +23,17 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunction} implementation for {@link SpatialExtentGeoShapeAggregator}. + * {@link AggregatorFunction} implementation for {@link SpatialExtentGeoShapeSourceValuesAggregator}. * This class is generated. Edit {@code AggregatorImplementer} instead. */ -public final class SpatialExtentGeoShapeAggregatorFunction implements AggregatorFunction { +public final class SpatialExtentGeoShapeSourceValuesAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final DriverContext driverContext; @@ -41,16 +41,16 @@ public final class SpatialExtentGeoShapeAggregatorFunction implements Aggregator private final List channels; - public SpatialExtentGeoShapeAggregatorFunction(DriverContext driverContext, + public SpatialExtentGeoShapeSourceValuesAggregatorFunction(DriverContext driverContext, List channels, SpatialExtentStateWrappedLongitudeState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; } - public static SpatialExtentGeoShapeAggregatorFunction create(DriverContext driverContext, - List channels) { - return new SpatialExtentGeoShapeAggregatorFunction(driverContext, channels, SpatialExtentGeoShapeAggregator.initSingle()); + public static SpatialExtentGeoShapeSourceValuesAggregatorFunction create( + DriverContext driverContext, List channels) { + return new SpatialExtentGeoShapeSourceValuesAggregatorFunction(driverContext, channels, SpatialExtentGeoShapeSourceValuesAggregator.initSingle()); } public static List intermediateStateDesc() { @@ -92,7 +92,7 @@ public void addRawInput(Page page, BooleanVector mask) { private void addRawVector(BytesRefVector vector) { BytesRef scratch = new BytesRef(); for (int i = 0; i < vector.getPositionCount(); i++) { - SpatialExtentGeoShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -102,7 +102,7 @@ private void addRawVector(BytesRefVector vector, BooleanVector mask) { if (mask.getBoolean(i) == false) { continue; } - SpatialExtentGeoShapeAggregator.combine(state, vector.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, vector.getBytesRef(i, scratch)); } } @@ -115,7 +115,7 @@ private void addRawBlock(BytesRefBlock block) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentGeoShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -132,7 +132,7 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - SpatialExtentGeoShapeAggregator.combine(state, block.getBytesRef(i, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, block.getBytesRef(i, scratch)); } } } @@ -141,43 +141,43 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - assert minNegX.getPositionCount() == 1; - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + assert top.getPositionCount() == 1; + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - assert minPosX.getPositionCount() == 1; - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + assert bottom.getPositionCount() == 1; + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - assert maxNegX.getPositionCount() == 1; - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + assert negLeft.getPositionCount() == 1; + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - assert maxPosX.getPositionCount() == 1; - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + assert negRight.getPositionCount() == 1; + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - assert maxY.getPositionCount() == 1; - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + assert posLeft.getPositionCount() == 1; + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minY.getPositionCount() == 1; - SpatialExtentGeoShapeAggregator.combineIntermediate(state, minNegX.getInt(0), minPosX.getInt(0), maxNegX.getInt(0), maxPosX.getInt(0), maxY.getInt(0), minY.getInt(0)); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert posRight.getPositionCount() == 1; + SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, top.getInt(0), bottom.getInt(0), negLeft.getInt(0), negRight.getInt(0), posLeft.getInt(0), posRight.getInt(0)); } @Override @@ -187,7 +187,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, DriverContext drive @Override public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { - blocks[offset] = SpatialExtentGeoShapeAggregator.evaluateFinal(state, driverContext); + blocks[offset] = SpatialExtentGeoShapeSourceValuesAggregator.evaluateFinal(state, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java similarity index 55% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunctionSupplier.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java index 19ad8726190ab..95aa4f3d30070 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier.java @@ -12,29 +12,30 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentCartesianShapeAggregator}. + * {@link AggregatorFunctionSupplier} implementation for {@link SpatialExtentGeoShapeSourceValuesAggregator}. * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ -public final class SpatialExtentCartesianShapeAggregatorFunctionSupplier implements AggregatorFunctionSupplier { +public final class SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier implements AggregatorFunctionSupplier { private final List channels; - public SpatialExtentCartesianShapeAggregatorFunctionSupplier(List channels) { + public SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier(List channels) { this.channels = channels; } @Override - public SpatialExtentCartesianShapeAggregatorFunction aggregator(DriverContext driverContext) { - return SpatialExtentCartesianShapeAggregatorFunction.create(driverContext, channels); + public SpatialExtentGeoShapeSourceValuesAggregatorFunction aggregator( + DriverContext driverContext) { + return SpatialExtentGeoShapeSourceValuesAggregatorFunction.create(driverContext, channels); } @Override - public SpatialExtentCartesianShapeGroupingAggregatorFunction groupingAggregator( + public SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction groupingAggregator( DriverContext driverContext) { - return SpatialExtentCartesianShapeGroupingAggregatorFunction.create(channels, driverContext); + return SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.create(channels, driverContext); } @Override public String describe() { - return "spatial_extent_cartesian of shapes"; + return "spatial_extent_geo_shape_source of valuess"; } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java similarity index 67% rename from x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeGroupingAggregatorFunction.java rename to x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java index ed03023c66427..c9be8deaf649c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction.java @@ -23,17 +23,17 @@ import org.elasticsearch.compute.operator.DriverContext; /** - * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentGeoShapeAggregator}. + * {@link GroupingAggregatorFunction} implementation for {@link SpatialExtentGeoShapeSourceValuesAggregator}. * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. */ -public final class SpatialExtentGeoShapeGroupingAggregatorFunction implements GroupingAggregatorFunction { +public final class SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("minNegX", ElementType.INT), - new IntermediateStateDesc("minPosX", ElementType.INT), - new IntermediateStateDesc("maxNegX", ElementType.INT), - new IntermediateStateDesc("maxPosX", ElementType.INT), - new IntermediateStateDesc("maxY", ElementType.INT), - new IntermediateStateDesc("minY", ElementType.INT) ); + new IntermediateStateDesc("top", ElementType.INT), + new IntermediateStateDesc("bottom", ElementType.INT), + new IntermediateStateDesc("negLeft", ElementType.INT), + new IntermediateStateDesc("negRight", ElementType.INT), + new IntermediateStateDesc("posLeft", ElementType.INT), + new IntermediateStateDesc("posRight", ElementType.INT) ); private final SpatialExtentGroupingStateWrappedLongitudeState state; @@ -41,16 +41,16 @@ public final class SpatialExtentGeoShapeGroupingAggregatorFunction implements Gr private final DriverContext driverContext; - public SpatialExtentGeoShapeGroupingAggregatorFunction(List channels, + public SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction(List channels, SpatialExtentGroupingStateWrappedLongitudeState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; } - public static SpatialExtentGeoShapeGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new SpatialExtentGeoShapeGroupingAggregatorFunction(channels, SpatialExtentGeoShapeAggregator.initGrouping(), driverContext); + public static SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction(channels, SpatialExtentGeoShapeSourceValuesAggregator.initGrouping(), driverContext); } public static List intermediateStateDesc() { @@ -114,7 +114,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefVector va BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } @@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); } } } @@ -159,7 +159,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val int groupEnd = groupStart + groups.getValueCount(groupPosition); for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); - SpatialExtentGeoShapeAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + SpatialExtentGeoShapeSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); } } } @@ -173,40 +173,40 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block minNegXUncast = page.getBlock(channels.get(0)); - if (minNegXUncast.areAllValuesNull()) { + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { return; } - IntVector minNegX = ((IntBlock) minNegXUncast).asVector(); - Block minPosXUncast = page.getBlock(channels.get(1)); - if (minPosXUncast.areAllValuesNull()) { + IntVector top = ((IntBlock) topUncast).asVector(); + Block bottomUncast = page.getBlock(channels.get(1)); + if (bottomUncast.areAllValuesNull()) { return; } - IntVector minPosX = ((IntBlock) minPosXUncast).asVector(); - Block maxNegXUncast = page.getBlock(channels.get(2)); - if (maxNegXUncast.areAllValuesNull()) { + IntVector bottom = ((IntBlock) bottomUncast).asVector(); + Block negLeftUncast = page.getBlock(channels.get(2)); + if (negLeftUncast.areAllValuesNull()) { return; } - IntVector maxNegX = ((IntBlock) maxNegXUncast).asVector(); - Block maxPosXUncast = page.getBlock(channels.get(3)); - if (maxPosXUncast.areAllValuesNull()) { + IntVector negLeft = ((IntBlock) negLeftUncast).asVector(); + Block negRightUncast = page.getBlock(channels.get(3)); + if (negRightUncast.areAllValuesNull()) { return; } - IntVector maxPosX = ((IntBlock) maxPosXUncast).asVector(); - Block maxYUncast = page.getBlock(channels.get(4)); - if (maxYUncast.areAllValuesNull()) { + IntVector negRight = ((IntBlock) negRightUncast).asVector(); + Block posLeftUncast = page.getBlock(channels.get(4)); + if (posLeftUncast.areAllValuesNull()) { return; } - IntVector maxY = ((IntBlock) maxYUncast).asVector(); - Block minYUncast = page.getBlock(channels.get(5)); - if (minYUncast.areAllValuesNull()) { + IntVector posLeft = ((IntBlock) posLeftUncast).asVector(); + Block posRightUncast = page.getBlock(channels.get(5)); + if (posRightUncast.areAllValuesNull()) { return; } - IntVector minY = ((IntBlock) minYUncast).asVector(); - assert minNegX.getPositionCount() == minPosX.getPositionCount() && minNegX.getPositionCount() == maxNegX.getPositionCount() && minNegX.getPositionCount() == maxPosX.getPositionCount() && minNegX.getPositionCount() == maxY.getPositionCount() && minNegX.getPositionCount() == minY.getPositionCount(); + IntVector posRight = ((IntBlock) posRightUncast).asVector(); + assert top.getPositionCount() == bottom.getPositionCount() && top.getPositionCount() == negLeft.getPositionCount() && top.getPositionCount() == negRight.getPositionCount() && top.getPositionCount() == posLeft.getPositionCount() && top.getPositionCount() == posRight.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - SpatialExtentGeoShapeAggregator.combineIntermediate(state, groupId, minNegX.getInt(groupPosition + positionOffset), minPosX.getInt(groupPosition + positionOffset), maxNegX.getInt(groupPosition + positionOffset), maxPosX.getInt(groupPosition + positionOffset), maxY.getInt(groupPosition + positionOffset), minY.getInt(groupPosition + positionOffset)); + SpatialExtentGeoShapeSourceValuesAggregator.combineIntermediate(state, groupId, top.getInt(groupPosition + positionOffset), bottom.getInt(groupPosition + positionOffset), negLeft.getInt(groupPosition + positionOffset), negRight.getInt(groupPosition + positionOffset), posLeft.getInt(groupPosition + positionOffset), posRight.getInt(groupPosition + positionOffset)); } } @@ -215,9 +215,9 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu if (input.getClass() != getClass()) { throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); } - SpatialExtentGroupingStateWrappedLongitudeState inState = ((SpatialExtentGeoShapeGroupingAggregatorFunction) input).state; + SpatialExtentGroupingStateWrappedLongitudeState inState = ((SpatialExtentGeoShapeSourceValuesGroupingAggregatorFunction) input).state; state.enableGroupIdTracking(new SeenGroupIds.Empty()); - SpatialExtentGeoShapeAggregator.combineStates(state, groupId, inState, position); + SpatialExtentGeoShapeSourceValuesAggregator.combineStates(state, groupId, inState, position); } @Override @@ -228,7 +228,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) @Override public void evaluateFinal(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - blocks[offset] = SpatialExtentGeoShapeAggregator.evaluateFinal(state, selected, driverContext); + blocks[offset] = SpatialExtentGeoShapeSourceValuesAggregator.evaluateFinal(state, selected, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java index 69df0fb8ceff1..2dfc60744be2e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java @@ -34,7 +34,8 @@ public EsqlRefCountingListener(ActionListener delegate) { } public ActionListener acquire() { - return refs.acquireListener().delegateResponse((l, e) -> { + var listener = ActionListener.assertAtLeastOnce(refs.acquireListener()); + return listener.delegateResponse((l, e) -> { failureCollector.unwrapAndCollect(e); l.onFailure(e); }); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st index 1cef234b2238f..6727cbb9408db 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st @@ -10,6 +10,7 @@ package org.elasticsearch.compute.aggregation; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; $endif$ +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigArrays; $if(BytesRef)$ import org.elasticsearch.common.util.BytesRefHash; @@ -267,64 +268,158 @@ $endif$ blocks[offset] = toBlock(driverContext.blockFactory(), selected); } + /** + * Builds a {@link Block} with the unique values collected for the {@code #selected} + * groups. This is the implementation of the final and intermediate results of the agg. + */ Block toBlock(BlockFactory blockFactory, IntVector selected) { if (values.size() == 0) { return blockFactory.newConstantNullBlock(selected.getPositionCount()); } -$if(BytesRef)$ - BytesRef scratch = new BytesRef(); + + long selectedCountsSize = 0; + long idsSize = 0; + try { + /* + * Get a count of all groups less than the maximum selected group. Count + * *downwards* so that we can flip the sign on all of the actually selected + * groups. Negative values in this array are always unselected groups. + */ + int selectedCountsLen = selected.max() + 1; + long adjust = RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + selectedCountsLen * Integer.BYTES + ); + blockFactory.adjustBreaker(adjust); + selectedCountsSize = adjust; + int[] selectedCounts = new int[selectedCountsLen]; + for (int id = 0; id < values.size(); id++) { +$if(long||BytesRef||double)$ + int group = (int) values.getKey1(id); +$elseif(float||int)$ + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); $endif$ - try ($Type$Block.Builder builder = blockFactory.new$Type$BlockBuilder(selected.getPositionCount())) { + if (group < selectedCounts.length) { + selectedCounts[group]--; + } + } + + /* + * Total the selected groups and turn the counts into the start index into a sort-of + * off-by-one running count. It's really the number of values that have been inserted + * into the results before starting on this group. Unselected groups will still + * have negative counts. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will contain 0, 3, -2, 4, 5 + */ + int total = 0; for (int s = 0; s < selected.getPositionCount(); s++) { - int selectedGroup = selected.getInt(s); - /* - * Count can effectively be in three states - 0, 1, many. We use those - * states to buffer the first value, so we can avoid calling - * beginPositionEntry on single valued fields. - */ - int count = 0; - $if(BytesRef)$long$else$$type$$endif$ first = 0; - for (int id = 0; id < values.size(); id++) { -$if(long||BytesRef)$ - if (values.getKey1(id) == selectedGroup) { - long value = values.getKey2(id); -$elseif(double)$ - if (values.getKey1(id) == selectedGroup) { - double value = Double.longBitsToDouble(values.getKey2(id)); -$elseif(float)$ - long both = values.get(id); - int group = (int) (both >>> Float.SIZE); - if (group == selectedGroup) { - float value = Float.intBitsToFloat((int) both); -$elseif(int)$ - long both = values.get(id); - int group = (int) (both >>> Integer.SIZE); - if (group == selectedGroup) { - int value = (int) both; + int group = selected.getInt(s); + int count = -selectedCounts[group]; + selectedCounts[group] = total; + total += count; + } + + /* + * Build a list of ids to insert in order *and* convert the running + * count in selectedCounts[group] into the end index (exclusive) in + * ids for each group. + * Here we use the negative counts to signal that a group hasn't been + * selected and the id containing values for that group is ignored. + * + * For example, if + * | Group | Value Count | Selected | + * |-------|-------------|----------| + * | 0 | 3 | <- | + * | 1 | 1 | <- | + * | 2 | 2 | | + * | 3 | 1 | <- | + * | 4 | 4 | <- | + * + * Then the total is 9 and the counts array will start with 0, 3, -2, 4, 5. + * The counts will end with 3, 4, -2, 5, 9. + */ + adjust = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + total * Integer.BYTES); + blockFactory.adjustBreaker(adjust); + idsSize = adjust; + int[] ids = new int[total]; + for (int id = 0; id < values.size(); id++) { +$if(long||BytesRef||double)$ + int group = (int) values.getKey1(id); +$elseif(float||int)$ + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); +$endif$ + if (group < selectedCounts.length && selectedCounts[group] >= 0) { + ids[selectedCounts[group]++] = id; + } + } + + /* + * Insert the ids in order. + */ +$if(BytesRef)$ + BytesRef scratch = new BytesRef(); $endif$ - switch (count) { - case 0 -> first = value; - case 1 -> { - builder.beginPositionEntry(); - builder.append$Type$($if(BytesRef)$bytes.get(first, scratch)$else$first$endif$); - builder.append$Type$($if(BytesRef)$bytes.get(value, scratch)$else$value$endif$); + try ($Type$Block.Builder builder = blockFactory.new$Type$BlockBuilder(selected.getPositionCount())) { + int start = 0; + for (int s = 0; s < selected.getPositionCount(); s++) { + int group = selected.getInt(s); + int end = selectedCounts[group]; + int count = end - start; + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> append(builder, ids[start]$if(BytesRef)$, scratch$endif$); + default -> { + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + append(builder, ids[i]$if(BytesRef)$, scratch$endif$); } - default -> builder.append$Type$($if(BytesRef)$bytes.get(value, scratch)$else$value$endif$); + builder.endPositionEntry(); } - count++; } + start = end; } - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> builder.append$Type$($if(BytesRef)$bytes.get(first, scratch)$else$first$endif$); - default -> builder.endPositionEntry(); - } + return builder.build(); } - return builder.build(); + } finally { + blockFactory.adjustBreaker(-selectedCountsSize - idsSize); } } - void enableGroupIdTracking(SeenGroupIds seen) { +$if(BytesRef)$ + private void append($Type$Block.Builder builder, int id, BytesRef scratch) { + BytesRef value = bytes.get(values.getKey2(id), scratch); + builder.appendBytesRef(value); + } + +$else$ + private void append($Type$Block.Builder builder, int id) { +$if(long)$ + long value = values.getKey2(id); +$elseif(double)$ + double value = Double.longBitsToDouble(values.getKey2(id)); +$elseif(float)$ + long both = values.get(id); + float value = Float.intBitsToFloat((int) both); +$elseif(int)$ + long both = values.get(id); + int value = (int) both; +$endif$ + builder.append$Type$(value); + } + +$endif$ + public void enableGroupIdTracking(SeenGroupIds seen) { // we figure out seen values from nulls on the values block } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java deleted file mode 100644 index 6bdd028f3d6ee..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/GeoPointEnvelopeVisitor.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.compute.aggregation.spatial; - -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude; - -class GeoPointEnvelopeVisitor extends SpatialEnvelopeVisitor.GeoPointVisitor { - GeoPointEnvelopeVisitor() { - super(WrapLongitude.WRAP); - } - - void reset() { - minY = Double.POSITIVE_INFINITY; - maxY = Double.NEGATIVE_INFINITY; - minNegX = Double.POSITIVE_INFINITY; - maxNegX = Double.NEGATIVE_INFINITY; - minPosX = Double.POSITIVE_INFINITY; - maxPosX = Double.NEGATIVE_INFINITY; - } - - double getMinNegX() { - return minNegX; - } - - double getMinPosX() { - return minPosX; - } - - double getMaxNegX() { - return maxNegX; - } - - double getMaxPosX() { - return maxPosX; - } - - double getMaxY() { - return maxY; - } - - double getMinY() { - return minY; - } - - static Rectangle asRectangle( - double minNegX, - double minPosX, - double maxNegX, - double maxPosX, - double maxY, - double minY, - WrapLongitude wrapLongitude - ) { - return SpatialEnvelopeVisitor.GeoPointVisitor.getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); - } -} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java index 6b29b20601dae..671ef6116ae6d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialAggregationUtils.java @@ -12,12 +12,10 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.GeometryValidator; -import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude; import org.elasticsearch.geometry.utils.WellKnownBinary; -class SpatialAggregationUtils { +public class SpatialAggregationUtils { private SpatialAggregationUtils() { /* Utility class */ } public static Geometry decode(BytesRef wkb) { @@ -52,26 +50,12 @@ public static double decodeLatitude(long encoded) { return GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)); } - public static int encodeNegativeLongitude(double d) { - return Double.isFinite(d) ? GeoEncodingUtils.encodeLongitude(d) : DEFAULT_NEG; + public static int encodeLongitude(double d) { + return Double.isFinite(d) ? GeoEncodingUtils.encodeLongitude(d) : encodeInfinity(d); } - public static int encodePositiveLongitude(double d) { - return Double.isFinite(d) ? GeoEncodingUtils.encodeLongitude(d) : DEFAULT_POS; - } - - public static Rectangle asRectangle(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { - assert minNegX <= 0 == maxNegX <= 0; - assert minPosX >= 0 == maxPosX >= 0; - return GeoPointEnvelopeVisitor.asRectangle( - minNegX <= 0 ? decodeLongitude(minNegX) : Double.POSITIVE_INFINITY, - minPosX >= 0 ? decodeLongitude(minPosX) : Double.POSITIVE_INFINITY, - maxNegX <= 0 ? decodeLongitude(maxNegX) : Double.NEGATIVE_INFINITY, - maxPosX >= 0 ? decodeLongitude(maxPosX) : Double.NEGATIVE_INFINITY, - GeoEncodingUtils.decodeLatitude(maxY), - GeoEncodingUtils.decodeLatitude(minY), - WrapLongitude.WRAP - ); + private static int encodeInfinity(double d) { + return d == Double.NEGATIVE_INFINITY ? Integer.MIN_VALUE : Integer.MAX_VALUE; } public static int maxNeg(int a, int b) { @@ -81,8 +65,4 @@ public static int maxNeg(int a, int b) { public static int minPos(int a, int b) { return a >= 0 && b >= 0 ? Math.min(a, b) : Math.max(a, b); } - - // The default values are intentionally non-negative/non-positive, so we can mark unassigned values. - public static final int DEFAULT_POS = -1; - public static final int DEFAULT_NEG = 1; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java index f64949b77707c..3a07754588566 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointDocValuesAggregator.java @@ -14,6 +14,9 @@ /** * Computes the extent of a set of cartesian points. It is assumed the points are encoded as longs. * This requires that the planner has planned that points are loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java index 3488af4525dcb..f7a74915f852e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianPointSourceValuesAggregator.java @@ -15,8 +15,9 @@ /** * Computes the extent of a set of cartesian points. It is assumed that the cartesian points are encoded as WKB BytesRef. * This requires that the planner has NOT planned that points are loaded from the index as doc-values, but from source instead. - * This is also used for final aggregations and aggregations in the coordinator node, - * even if the local node partial aggregation is done with {@link SpatialExtentCartesianPointDocValuesAggregator}. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java new file mode 100644 index 0000000000000..1305139ab2c29 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeDocValuesAggregator.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation.spatial; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +/** + * Computes the extent of a set of cartesian shapes read from doc-values, which means they are encoded as an array of integers. + * This requires that the planner has planned that the shape extent is loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. + */ +@Aggregator( + { + @IntermediateState(name = "minX", type = "INT"), + @IntermediateState(name = "maxX", type = "INT"), + @IntermediateState(name = "maxY", type = "INT"), + @IntermediateState(name = "minY", type = "INT") } +) +@GroupingAggregator +class SpatialExtentCartesianShapeDocValuesAggregator extends SpatialExtentAggregator { + public static SpatialExtentState initSingle() { + return new SpatialExtentState(PointType.CARTESIAN); + } + + public static SpatialExtentGroupingState initGrouping() { + return new SpatialExtentGroupingState(PointType.CARTESIAN); + } + + public static void combine(SpatialExtentState current, int[] values) { + current.add(values); + } + + public static void combine(SpatialExtentGroupingState current, int groupId, int[] values) { + current.add(groupId, values); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java similarity index 67% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregator.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java index 6d50d27aa5a2d..adcf072fbddd2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentCartesianShapeSourceValuesAggregator.java @@ -13,8 +13,11 @@ import org.elasticsearch.compute.ann.IntermediateState; /** - * Computes the extent of a set of cartesian shapes. It is assumed that the cartesian shapes are encoded as WKB BytesRef. - * We do not currently support reading shape values or extents from doc values. + * Computes the extent of a set of cartesian shapes read from source, which means they are encoded as WKB BytesRef. + * This requires that the planner has NOT planned that shapes are loaded from the index as doc-values, but from source instead. + * The intermediate state is the extent of the shapes, encoded as four integers: minX, maxX, maxY, minY. + * The order of the integers is the same as defined in the constructor of the Rectangle class. + * Note that this is very different from the six values used for the intermediate state of geo_shape geometries. */ @Aggregator( { @@ -24,7 +27,7 @@ @IntermediateState(name = "minY", type = "INT") } ) @GroupingAggregator -class SpatialExtentCartesianShapeAggregator extends SpatialExtentAggregator { +class SpatialExtentCartesianShapeSourceValuesAggregator extends SpatialExtentAggregator { public static SpatialExtentState initSingle() { return new SpatialExtentState(PointType.CARTESIAN); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java index b9b8bf65e116b..93008d4ee4ff0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointDocValuesAggregator.java @@ -14,15 +14,19 @@ /** * Computes the extent of a set of geo points. It is assumed the points are encoded as longs. * This requires that the planner has planned that points are loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator class SpatialExtentGeoPointDocValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java index 36a4e359f23fc..d454b40b1a44f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoPointSourceValuesAggregator.java @@ -15,17 +15,19 @@ /** * Computes the extent of a set of geo points. It is assumed that the geo points are encoded as WKB BytesRef. * This requires that the planner has NOT planned that points are loaded from the index as doc-values, but from source instead. - * This is also used for final aggregations and aggregations in the coordinator node, - * even if the local node partial aggregation is done with {@link SpatialExtentGeoPointDocValuesAggregator}. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator class SpatialExtentGeoPointSourceValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java new file mode 100644 index 0000000000000..26f8ae156aacc --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeDocValuesAggregator.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation.spatial; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +/** + * Computes the extent of a set of geo shapes read from doc-values, which means they are encoded as an array of integers. + * This requires that the planner has planned that the shape extent is loaded from the index as doc-values. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. + */ +@Aggregator( + { + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } +) +@GroupingAggregator +class SpatialExtentGeoShapeDocValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { + public static SpatialExtentStateWrappedLongitudeState initSingle() { + return new SpatialExtentStateWrappedLongitudeState(); + } + + public static SpatialExtentGroupingStateWrappedLongitudeState initGrouping() { + return new SpatialExtentGroupingStateWrappedLongitudeState(); + } + + public static void combine(SpatialExtentStateWrappedLongitudeState current, int[] values) { + current.add(values); + } + + public static void combine(SpatialExtentGroupingStateWrappedLongitudeState current, int groupId, int[] values) { + current.add(groupId, values); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java similarity index 52% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregator.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java index 3d1b9b6300c9d..cda0aedfb3ae4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGeoShapeSourceValuesAggregator.java @@ -13,21 +13,24 @@ import org.elasticsearch.compute.ann.IntermediateState; /** - * Computes the extent of a set of geo shapes. It is assumed that the geo shapes are encoded as WKB BytesRef. - * We do not currently support reading shape values or extents from doc values. + * Computes the extent of a set of geo shapes read from source, which means they are encoded as WKB BytesRef. + * This requires that the planner has NOT planned that shapes are loaded from the index as doc-values, but from source instead. + * The intermediate state is the extent of the shapes, encoded as six integers: top, bottom, negLeft, negRight, posLeft, posRight. + * The order of the integers is the same as defined in the constructor of the Extent class, + * as that is the order in which the values are stored in shape doc-values. + * Note that this is very different from the four values used for the intermediate state of cartesian_shape geometries. */ @Aggregator( { - @IntermediateState(name = "minNegX", type = "INT"), - @IntermediateState(name = "minPosX", type = "INT"), - @IntermediateState(name = "maxNegX", type = "INT"), - @IntermediateState(name = "maxPosX", type = "INT"), - @IntermediateState(name = "maxY", type = "INT"), - @IntermediateState(name = "minY", type = "INT") } + @IntermediateState(name = "top", type = "INT"), + @IntermediateState(name = "bottom", type = "INT"), + @IntermediateState(name = "negLeft", type = "INT"), + @IntermediateState(name = "negRight", type = "INT"), + @IntermediateState(name = "posLeft", type = "INT"), + @IntermediateState(name = "posRight", type = "INT") } ) @GroupingAggregator -class SpatialExtentGeoShapeAggregator extends SpatialExtentLongitudeWrappingAggregator { - // TODO support non-longitude wrapped geo shapes. +class SpatialExtentGeoShapeSourceValuesAggregator extends SpatialExtentLongitudeWrappingAggregator { public static SpatialExtentStateWrappedLongitudeState initSingle() { return new SpatialExtentStateWrappedLongitudeState(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java index cb765e4d6757e..9fb548dceaad9 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingState.java @@ -18,6 +18,7 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; @@ -53,11 +54,18 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected, Drive ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - assert hasValue(group); - minXsBuilder.appendInt(minXs.get(group)); - maxXsBuilder.appendInt(maxXs.get(group)); - maxYsBuilder.appendInt(maxYs.get(group)); - minYsBuilder.appendInt(minYs.get(group)); + if (hasValue(group)) { + minXsBuilder.appendInt(minXs.get(group)); + maxXsBuilder.appendInt(maxXs.get(group)); + maxYsBuilder.appendInt(maxYs.get(group)); + minYsBuilder.appendInt(minYs.get(group)); + } else { + // TODO: Should we add Nulls here instead? + minXsBuilder.appendInt(Integer.MAX_VALUE); + maxXsBuilder.appendInt(Integer.MIN_VALUE); + maxYsBuilder.appendInt(Integer.MIN_VALUE); + minYsBuilder.appendInt(Integer.MAX_VALUE); + } } blocks[offset + 0] = minXsBuilder.build(); blocks[offset + 1] = maxXsBuilder.build(); @@ -66,6 +74,32 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected, Drive } } + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int groupId, int[] values) { + if (values.length == 6) { + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(groupId, Math.min(negLeft, posLeft), Math.max(negRight, posRight), top, bottom); + } else if (values.length == 4) { + // Values are stored according to the order defined in the Rectangle class + int minX = values[0]; + int maxX = values[1]; + int maxY = values[2]; + int minY = values[3]; + add(groupId, minX, maxX, maxY, minY); + } else { + throw new IllegalArgumentException("Expected 4 or 6 values, got " + values.length); + } + } + public void add(int groupId, Geometry geometry) { ensureCapacity(groupId); pointType.computeEnvelope(geometry) @@ -80,6 +114,10 @@ public void add(int groupId, Geometry geometry) { ); } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(int groupId, long encoded) { int x = pointType.extractX(encoded); int y = pointType.extractY(encoded); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java index 41bc50abcf6bc..9f8fca5236d14 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentGroupingStateWrappedLongitudeState.java @@ -19,20 +19,23 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; +import static org.elasticsearch.compute.aggregation.spatial.SpatialExtentStateWrappedLongitudeState.asRectangle; + final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArrayState implements GroupingAggregatorState { // Only geo points support longitude wrapping. private static final PointType POINT_TYPE = PointType.GEO; - private IntArray minNegXs; - private IntArray minPosXs; - private IntArray maxNegXs; - private IntArray maxPosXs; - private IntArray maxYs; - private IntArray minYs; + private IntArray tops; + private IntArray bottoms; + private IntArray negLefts; + private IntArray negRights; + private IntArray posLefts; + private IntArray posRights; - private GeoPointEnvelopeVisitor geoPointVisitor = new GeoPointEnvelopeVisitor(); + private final SpatialEnvelopeVisitor.GeoPointVisitor geoPointVisitor; SpatialExtentGroupingStateWrappedLongitudeState() { this(BigArrays.NON_RECYCLING_INSTANCE); @@ -40,44 +43,52 @@ final class SpatialExtentGroupingStateWrappedLongitudeState extends AbstractArra SpatialExtentGroupingStateWrappedLongitudeState(BigArrays bigArrays) { super(bigArrays); - this.minNegXs = bigArrays.newIntArray(0, false); - this.minPosXs = bigArrays.newIntArray(0, false); - this.maxNegXs = bigArrays.newIntArray(0, false); - this.maxPosXs = bigArrays.newIntArray(0, false); - this.maxYs = bigArrays.newIntArray(0, false); - this.minYs = bigArrays.newIntArray(0, false); + this.tops = bigArrays.newIntArray(0, false); + this.bottoms = bigArrays.newIntArray(0, false); + this.negLefts = bigArrays.newIntArray(0, false); + this.negRights = bigArrays.newIntArray(0, false); + this.posLefts = bigArrays.newIntArray(0, false); + this.posRights = bigArrays.newIntArray(0, false); enableGroupIdTracking(new SeenGroupIds.Empty()); + this.geoPointVisitor = new SpatialEnvelopeVisitor.GeoPointVisitor(SpatialEnvelopeVisitor.WrapLongitude.WRAP); } @Override public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { assert blocks.length >= offset; try ( - var minNegXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var minPosXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var maxNegXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var maxPosXsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var maxYsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); - var minYsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var topsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var bottomsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var negLeftsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var negRightsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var posLeftsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var posRightsBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - assert hasValue(group); - assert minNegXs.get(group) <= 0 == maxNegXs.get(group) <= 0; - assert minPosXs.get(group) >= 0 == maxPosXs.get(group) >= 0; - minNegXsBuilder.appendInt(minNegXs.get(group)); - minPosXsBuilder.appendInt(minPosXs.get(group)); - maxNegXsBuilder.appendInt(maxNegXs.get(group)); - maxPosXsBuilder.appendInt(maxPosXs.get(group)); - maxYsBuilder.appendInt(maxYs.get(group)); - minYsBuilder.appendInt(minYs.get(group)); + if (hasValue(group)) { + topsBuilder.appendInt(tops.get(group)); + bottomsBuilder.appendInt(bottoms.get(group)); + negLeftsBuilder.appendInt(negLefts.get(group)); + negRightsBuilder.appendInt(negRights.get(group)); + posLeftsBuilder.appendInt(posLefts.get(group)); + posRightsBuilder.appendInt(posRights.get(group)); + } else { + // TODO: Should we add Nulls here instead? + topsBuilder.appendInt(Integer.MIN_VALUE); + bottomsBuilder.appendInt(Integer.MAX_VALUE); + negLeftsBuilder.appendInt(Integer.MAX_VALUE); + negRightsBuilder.appendInt(Integer.MIN_VALUE); + posLeftsBuilder.appendInt(Integer.MAX_VALUE); + posRightsBuilder.appendInt(Integer.MIN_VALUE); + } } - blocks[offset + 0] = minNegXsBuilder.build(); - blocks[offset + 1] = minPosXsBuilder.build(); - blocks[offset + 2] = maxNegXsBuilder.build(); - blocks[offset + 3] = maxPosXsBuilder.build(); - blocks[offset + 4] = maxYsBuilder.build(); - blocks[offset + 5] = minYsBuilder.build(); + blocks[offset + 0] = topsBuilder.build(); + blocks[offset + 1] = bottomsBuilder.build(); + blocks[offset + 2] = negLeftsBuilder.build(); + blocks[offset + 3] = negRightsBuilder.build(); + blocks[offset + 4] = posLeftsBuilder.build(); + blocks[offset + 5] = posRightsBuilder.build(); } } @@ -87,12 +98,12 @@ public void add(int groupId, Geometry geo) { if (geo.visit(new SpatialEnvelopeVisitor(geoPointVisitor))) { add( groupId, - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMinNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMinPosX()), - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMaxNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMaxPosX()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY()) + POINT_TYPE.encoder().encodeY(geoPointVisitor.getTop()), + POINT_TYPE.encoder().encodeY(geoPointVisitor.getBottom()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegRight()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosRight()) ); } } @@ -102,53 +113,73 @@ public void add(int groupId, SpatialExtentGroupingStateWrappedLongitudeState inS if (inState.hasValue(inPosition)) { add( groupId, - inState.minNegXs.get(inPosition), - inState.minPosXs.get(inPosition), - inState.maxNegXs.get(inPosition), - inState.maxPosXs.get(inPosition), - inState.maxYs.get(inPosition), - inState.minYs.get(inPosition) + inState.tops.get(inPosition), + inState.bottoms.get(inPosition), + inState.negLefts.get(inPosition), + inState.negRights.get(inPosition), + inState.posLefts.get(inPosition), + inState.posRights.get(inPosition) ); } } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(int groupId, long encoded) { int x = POINT_TYPE.extractX(encoded); int y = POINT_TYPE.extractY(encoded); - add(groupId, x, x, x, x, y, y); + add(groupId, y, y, x, x, x, x); + } + + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int groupId, int[] values) { + if (values.length != 6) { + throw new IllegalArgumentException("Expected 6 values, got " + values.length); + } + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(groupId, top, bottom, negLeft, negRight, posLeft, posRight); } - public void add(int groupId, int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { + public void add(int groupId, int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { ensureCapacity(groupId); if (hasValue(groupId)) { - minNegXs.set(groupId, Math.min(minNegXs.get(groupId), minNegX)); - minPosXs.set(groupId, SpatialAggregationUtils.minPos(minPosXs.get(groupId), minPosX)); - maxNegXs.set(groupId, SpatialAggregationUtils.maxNeg(maxNegXs.get(groupId), maxNegX)); - maxPosXs.set(groupId, Math.max(maxPosXs.get(groupId), maxPosX)); - maxYs.set(groupId, Math.max(maxYs.get(groupId), maxY)); - minYs.set(groupId, Math.min(minYs.get(groupId), minY)); + tops.set(groupId, Math.max(tops.get(groupId), top)); + bottoms.set(groupId, Math.min(bottoms.get(groupId), bottom)); + negLefts.set(groupId, Math.min(negLefts.get(groupId), negLeft)); + negRights.set(groupId, SpatialAggregationUtils.maxNeg(negRights.get(groupId), negRight)); + posLefts.set(groupId, SpatialAggregationUtils.minPos(posLefts.get(groupId), posLeft)); + posRights.set(groupId, Math.max(posRights.get(groupId), posRight)); } else { - minNegXs.set(groupId, minNegX); - minPosXs.set(groupId, minPosX); - maxNegXs.set(groupId, maxNegX); - maxPosXs.set(groupId, maxPosX); - maxYs.set(groupId, maxY); - minYs.set(groupId, minY); + tops.set(groupId, top); + bottoms.set(groupId, bottom); + negLefts.set(groupId, negLeft); + negRights.set(groupId, negRight); + posLefts.set(groupId, posLeft); + posRights.set(groupId, posRight); } - assert minNegX <= 0 == maxNegX <= 0 : "minNegX=" + minNegX + " maxNegX=" + maxNegX; - assert minPosX >= 0 == maxPosX >= 0 : "minPosX=" + minPosX + " maxPosX=" + maxPosX; trackGroupId(groupId); } private void ensureCapacity(int groupId) { long requiredSize = groupId + 1; - if (minNegXs.size() < requiredSize) { - minNegXs = bigArrays.grow(minNegXs, requiredSize); - minPosXs = bigArrays.grow(minPosXs, requiredSize); - maxNegXs = bigArrays.grow(maxNegXs, requiredSize); - maxPosXs = bigArrays.grow(maxPosXs, requiredSize); - minYs = bigArrays.grow(minYs, requiredSize); - maxYs = bigArrays.grow(maxYs, requiredSize); + if (negLefts.size() < requiredSize) { + tops = bigArrays.grow(tops, requiredSize); + bottoms = bigArrays.grow(bottoms, requiredSize); + negLefts = bigArrays.grow(negLefts, requiredSize); + negRights = bigArrays.grow(negRights, requiredSize); + posLefts = bigArrays.grow(posLefts, requiredSize); + posRights = bigArrays.grow(posRights, requiredSize); } } @@ -160,13 +191,13 @@ public Block toBlock(IntVector selected, DriverContext driverContext) { builder.appendBytesRef( new BytesRef( WellKnownBinary.toWKB( - SpatialAggregationUtils.asRectangle( - minNegXs.get(si), - minPosXs.get(si), - maxNegXs.get(si), - maxPosXs.get(si), - maxYs.get(si), - minYs.get(si) + asRectangle( + tops.get(si), + bottoms.get(si), + negLefts.get(si), + negRights.get(si), + posLefts.get(si), + posRights.get(si) ), ByteOrder.LITTLE_ENDIAN ) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java index 80ba2d5e45658..2d89ba78d1025 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentLongitudeWrappingAggregator.java @@ -16,27 +16,27 @@ abstract class SpatialExtentLongitudeWrappingAggregator { public static void combineIntermediate( SpatialExtentStateWrappedLongitudeState current, - int minNegX, - int minPosX, - int maxNegX, - int maxPosX, - int maxY, - int minY + int top, + int bottom, + int negLeft, + int negRight, + int posLeft, + int posRight ) { - current.add(minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + current.add(top, bottom, negLeft, negRight, posLeft, posRight); } public static void combineIntermediate( SpatialExtentGroupingStateWrappedLongitudeState current, int groupId, - int minNegX, - int minPosX, - int maxNegX, - int maxPosX, - int maxY, - int minY + int top, + int bottom, + int negLeft, + int negRight, + int posLeft, + int posRight ) { - current.add(groupId, minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + current.add(groupId, top, bottom, negLeft, negRight, posLeft, posRight); } public static Block evaluateFinal(SpatialExtentStateWrappedLongitudeState state, DriverContext driverContext) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java index 3dc150d1702a2..cd52d346b09f4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentState.java @@ -15,6 +15,7 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.WellKnownBinary; import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; @@ -55,6 +56,32 @@ public void add(Geometry geo) { ); } + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int[] values) { + if (values.length == 6) { + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(Math.min(negLeft, posLeft), Math.max(negRight, posRight), top, bottom); + } else if (values.length == 4) { + // Values are stored according to the order defined in the Rectangle class + int minX = values[0]; + int maxX = values[1]; + int maxY = values[2]; + int minY = values[3]; + add(minX, maxX, maxY, minY); + } else { + throw new IllegalArgumentException("Expected 4 or 6 values, got " + values.length); + } + } + public void add(int minX, int maxX, int maxY, int minY) { seen = true; this.minX = Math.min(this.minX, minX); @@ -63,6 +90,10 @@ public void add(int minX, int maxX, int maxY, int minY) { this.minY = Math.min(this.minY, minY); } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(long encoded) { int x = pointType.extractX(encoded); int y = pointType.extractY(encoded); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java index 0d6163636fcde..86b41b5b8359c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/SpatialExtentStateWrappedLongitudeState.java @@ -7,28 +7,35 @@ package org.elasticsearch.compute.aggregation.spatial; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.aggregation.AggregatorState; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import java.nio.ByteOrder; +import static org.elasticsearch.compute.aggregation.spatial.SpatialAggregationUtils.decodeLongitude; + final class SpatialExtentStateWrappedLongitudeState implements AggregatorState { // Only geo points support longitude wrapping. private static final PointType POINT_TYPE = PointType.GEO; private boolean seen = false; - private int minNegX = SpatialAggregationUtils.DEFAULT_NEG; - private int minPosX = SpatialAggregationUtils.DEFAULT_POS; - private int maxNegX = SpatialAggregationUtils.DEFAULT_NEG; - private int maxPosX = SpatialAggregationUtils.DEFAULT_POS; - private int maxY = Integer.MIN_VALUE; - private int minY = Integer.MAX_VALUE; + private int top = Integer.MIN_VALUE; + private int bottom = Integer.MAX_VALUE; + private int negLeft = Integer.MAX_VALUE; + private int negRight = Integer.MIN_VALUE; + private int posLeft = Integer.MAX_VALUE; + private int posRight = Integer.MIN_VALUE; - private GeoPointEnvelopeVisitor geoPointVisitor = new GeoPointEnvelopeVisitor(); + private final SpatialEnvelopeVisitor.GeoPointVisitor geoPointVisitor = new SpatialEnvelopeVisitor.GeoPointVisitor( + SpatialEnvelopeVisitor.WrapLongitude.WRAP + ); @Override public void close() {} @@ -37,44 +44,64 @@ public void close() {} public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { assert blocks.length >= offset + 6; var blockFactory = driverContext.blockFactory(); - blocks[offset + 0] = blockFactory.newConstantIntBlockWith(minNegX, 1); - blocks[offset + 1] = blockFactory.newConstantIntBlockWith(minPosX, 1); - blocks[offset + 2] = blockFactory.newConstantIntBlockWith(maxNegX, 1); - blocks[offset + 3] = blockFactory.newConstantIntBlockWith(maxPosX, 1); - blocks[offset + 4] = blockFactory.newConstantIntBlockWith(maxY, 1); - blocks[offset + 5] = blockFactory.newConstantIntBlockWith(minY, 1); + blocks[offset + 0] = blockFactory.newConstantIntBlockWith(top, 1); + blocks[offset + 1] = blockFactory.newConstantIntBlockWith(bottom, 1); + blocks[offset + 2] = blockFactory.newConstantIntBlockWith(negLeft, 1); + blocks[offset + 3] = blockFactory.newConstantIntBlockWith(negRight, 1); + blocks[offset + 4] = blockFactory.newConstantIntBlockWith(posLeft, 1); + blocks[offset + 5] = blockFactory.newConstantIntBlockWith(posRight, 1); } public void add(Geometry geo) { geoPointVisitor.reset(); if (geo.visit(new SpatialEnvelopeVisitor(geoPointVisitor))) { add( - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMinNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMinPosX()), - SpatialAggregationUtils.encodeNegativeLongitude(geoPointVisitor.getMaxNegX()), - SpatialAggregationUtils.encodePositiveLongitude(geoPointVisitor.getMaxPosX()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMaxY()), - POINT_TYPE.encoder().encodeY(geoPointVisitor.getMinY()) + POINT_TYPE.encoder().encodeY(geoPointVisitor.getTop()), + POINT_TYPE.encoder().encodeY(geoPointVisitor.getBottom()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getNegRight()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosLeft()), + SpatialAggregationUtils.encodeLongitude(geoPointVisitor.getPosRight()) ); } } - public void add(int minNegX, int minPosX, int maxNegX, int maxPosX, int maxY, int minY) { + /** + * This method is used when extents are extracted from the doc-values field by the {@link GeometryDocValueReader}. + * This optimization is enabled when the field has doc-values and is only used in the ST_EXTENT aggregation. + */ + public void add(int[] values) { + if (values.length != 6) { + throw new IllegalArgumentException("Expected 6 values, got " + values.length); + } + // Values are stored according to the order defined in the Extent class + int top = values[0]; + int bottom = values[1]; + int negLeft = values[2]; + int negRight = values[3]; + int posLeft = values[4]; + int posRight = values[5]; + add(top, bottom, negLeft, negRight, posLeft, posRight); + } + + public void add(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { seen = true; - this.minNegX = Math.min(this.minNegX, minNegX); - this.minPosX = SpatialAggregationUtils.minPos(this.minPosX, minPosX); - this.maxNegX = SpatialAggregationUtils.maxNeg(this.maxNegX, maxNegX); - this.maxPosX = Math.max(this.maxPosX, maxPosX); - this.maxY = Math.max(this.maxY, maxY); - this.minY = Math.min(this.minY, minY); - assert this.minNegX <= 0 == this.maxNegX <= 0 : "minNegX=" + this.minNegX + " maxNegX=" + this.maxNegX; - assert this.minPosX >= 0 == this.maxPosX >= 0 : "minPosX=" + this.minPosX + " maxPosX=" + this.maxPosX; + this.top = Math.max(this.top, top); + this.bottom = Math.min(this.bottom, bottom); + this.negLeft = Math.min(this.negLeft, negLeft); + this.negRight = SpatialAggregationUtils.maxNeg(this.negRight, negRight); + this.posLeft = SpatialAggregationUtils.minPos(this.posLeft, posLeft); + this.posRight = Math.max(this.posRight, posRight); } + /** + * This method is used when the field is a geo_point or cartesian_point and is loaded from doc-values. + * This optimization is enabled when the field has doc-values and is only used in a spatial aggregation. + */ public void add(long encoded) { int x = POINT_TYPE.extractX(encoded); int y = POINT_TYPE.extractY(encoded); - add(x, x, x, x, y, y); + add(y, y, x, x, x, x); } public Block toBlock(DriverContext driverContext) { @@ -83,9 +110,18 @@ public Block toBlock(DriverContext driverContext) { } private byte[] toWKB() { - return WellKnownBinary.toWKB( - SpatialAggregationUtils.asRectangle(minNegX, minPosX, maxNegX, maxPosX, maxY, minY), - ByteOrder.LITTLE_ENDIAN + return WellKnownBinary.toWKB(asRectangle(top, bottom, negLeft, negRight, posLeft, posRight), ByteOrder.LITTLE_ENDIAN); + } + + static Rectangle asRectangle(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + return SpatialEnvelopeVisitor.GeoPointVisitor.getResult( + GeoEncodingUtils.decodeLatitude(top), + GeoEncodingUtils.decodeLatitude(bottom), + negLeft <= 0 ? decodeLongitude(negLeft) : Double.POSITIVE_INFINITY, + negRight <= 0 ? decodeLongitude(negRight) : Double.NEGATIVE_INFINITY, + posLeft >= 0 ? decodeLongitude(posLeft) : Double.POSITIVE_INFINITY, + posRight >= 0 ? decodeLongitude(posRight) : Double.NEGATIVE_INFINITY, + SpatialEnvelopeVisitor.WrapLongitude.WRAP ); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java index 6dca94b9bc79a..63d79a9198622 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java @@ -8,10 +8,12 @@ package org.elasticsearch.compute.data.sort; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -29,6 +31,11 @@ /** * Aggregates the top N variable length {@link BytesRef} values per bucket. * See {@link BucketedSort} for more information. + *

+ * This is substantially different from {@link IpBucketedSort} because + * this has to handle variable length byte strings. To do that it allocates + * a heap of {@link BreakingBytesRefBuilder}s. + *

*/ public class BytesRefBucketedSort implements Releasable { private final BucketedSortCommon common; @@ -123,7 +130,7 @@ public void collect(BytesRef value, int bucket) { // Gathering mode long requiredSize = common.endIndex(rootIndex); if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); common.assertValidNextOffset(next); @@ -271,13 +278,23 @@ private void swap(long lhs, long rhs) { /** * Allocate storage for more buckets and store the "next gather offset" - * for those new buckets. + * for those new buckets. We always grow the storage by whole bucket's + * worth of slots at a time. We never allocate space for partial buckets. */ - private void grow(long requiredSize) { + private void grow(int bucket) { long oldMax = values.size(); - values = common.bigArrays.grow(values, requiredSize); + assert oldMax % common.bucketSize == 0; + + long newSize = BigArrays.overSize( + ((long) bucket + 1) * common.bucketSize, + PageCacheRecycler.OBJECT_PAGE_SIZE, + RamUsageEstimator.NUM_BYTES_OBJECT_REF + ); + // Round up to the next full bucket. + newSize = (newSize + common.bucketSize - 1) / common.bucketSize; + values = common.bigArrays.resize(values, newSize * common.bucketSize); // Set the next gather offsets for all newly allocated buckets. - fillGatherOffsets(oldMax - (oldMax % common.bucketSize)); + fillGatherOffsets(oldMax); } /** @@ -296,6 +313,7 @@ private void fillGatherOffsets(long startingAt) { bytes.grow(Integer.BYTES); bytes.setLength(Integer.BYTES); ByteUtils.writeIntLE(nextOffset, bytes.bytes(), 0); + checkInvariant(Math.toIntExact(bucketRoot / common.bucketSize)); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java index 4eb31ea30db22..4392d3994886c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -26,6 +27,11 @@ /** * Aggregates the top N IP values per bucket. * See {@link BucketedSort} for more information. + *

+ * This is substantially different from {@link BytesRefBucketedSort} because + * this takes advantage of IPs having a fixed length and allocates a dense + * storage for them. + *

*/ public class IpBucketedSort implements Releasable { private static final int IP_LENGTH = 16; // Bytes. It's ipv6. @@ -101,7 +107,7 @@ public void collect(BytesRef value, int bucket) { // Gathering mode long requiredSize = common.endIndex(rootIndex) * IP_LENGTH; if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); common.assertValidNextOffset(next); @@ -268,17 +274,23 @@ private void swap(long lhs, long rhs) { * Allocate storage for more buckets and store the "next gather offset" * for those new buckets. */ - private void grow(long minSize) { + private void grow(int bucket) { long oldMax = values.size() / IP_LENGTH; - values = common.bigArrays.grow(values, minSize); + assert oldMax % common.bucketSize == 0; + + int bucketBytes = common.bucketSize * IP_LENGTH; + long newSize = BigArrays.overSize(((long) bucket + 1) * bucketBytes, PageCacheRecycler.BYTE_PAGE_SIZE, 1); + // Round up to the next full bucket. + newSize = (newSize + bucketBytes - 1) / bucketBytes; + values = common.bigArrays.resize(values, newSize * bucketBytes); // Set the next gather offsets for all newly allocated buckets. - setNextGatherOffsets(oldMax - (oldMax % common.bucketSize)); + fillGatherOffsets(oldMax); } /** * Maintain the "next gather offsets" for newly allocated buckets. */ - private void setNextGatherOffsets(long startingAt) { + private void fillGatherOffsets(long startingAt) { int nextOffset = common.bucketSize - 1; for (long bucketRoot = startingAt; bucketRoot < values.size() / IP_LENGTH; bucketRoot += common.bucketSize) { setNextGatherOffset(bucketRoot, nextOffset); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st index 6587743e34b6f..095d48021e9c1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st @@ -10,6 +10,7 @@ package org.elasticsearch.compute.data.sort; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.common.util.$Type$Array; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntVector; @@ -101,7 +102,7 @@ public class $Type$BucketedSort implements Releasable { // Gathering mode long requiredSize = rootIndex + bucketSize; if (values.size() < requiredSize) { - grow(requiredSize); + grow(bucket); } int next = getNextGatherOffset(rootIndex); assert 0 <= next && next < bucketSize @@ -261,19 +262,25 @@ $endif$ /** * Allocate storage for more buckets and store the "next gather offset" - * for those new buckets. + * for those new buckets. We always grow the storage by whole bucket's + * worth of slots at a time. We never allocate space for partial buckets. */ - private void grow(long minSize) { + private void grow(int bucket) { long oldMax = values.size(); - values = bigArrays.grow(values, minSize); + assert oldMax % bucketSize == 0; + + long newSize = BigArrays.overSize(((long) bucket + 1) * bucketSize, PageCacheRecycler.$TYPE$_PAGE_SIZE, $BYTES$); + // Round up to the next full bucket. + newSize = (newSize + bucketSize - 1) / bucketSize; + values = bigArrays.resize(values, newSize * bucketSize); // Set the next gather offsets for all newly allocated buckets. - setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + fillGatherOffsets(oldMax); } /** * Maintain the "next gather offsets" for newly allocated buckets. */ - private void setNextGatherOffsets(long startingAt) { + private void fillGatherOffsets(long startingAt) { int nextOffset = getBucketSize() - 1; for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { setNextGatherOffset(bucketRoot, nextOffset); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java index 3d34067e1a839..76f0fb0167b86 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java @@ -16,10 +16,10 @@ import org.elasticsearch.compute.data.DocBlock; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.DoubleVector; -import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.Limiter; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.core.Releasables; @@ -37,6 +37,7 @@ public class LuceneSourceOperator extends LuceneOperator { private int currentPagePos = 0; private int remainingDocs; + private final Limiter limiter; private IntVector.Builder docsBuilder; private DoubleVector.Builder scoreBuilder; @@ -46,6 +47,7 @@ public class LuceneSourceOperator extends LuceneOperator { public static class Factory extends LuceneOperator.Factory { private final int maxPageSize; + private final Limiter limiter; public Factory( List contexts, @@ -58,11 +60,13 @@ public Factory( ) { super(contexts, queryFunction, dataPartitioning, taskConcurrency, limit, scoring ? COMPLETE : COMPLETE_NO_SCORES); this.maxPageSize = maxPageSize; + // TODO: use a single limiter for multiple stage execution + this.limiter = limit == NO_LIMIT ? Limiter.NO_LIMIT : new Limiter(limit); } @Override public SourceOperator get(DriverContext driverContext) { - return new LuceneSourceOperator(driverContext.blockFactory(), maxPageSize, sliceQueue, limit, scoreMode); + return new LuceneSourceOperator(driverContext.blockFactory(), maxPageSize, sliceQueue, limit, limiter, scoreMode); } public int maxPageSize() { @@ -84,10 +88,18 @@ public String describe() { } @SuppressWarnings("this-escape") - public LuceneSourceOperator(BlockFactory blockFactory, int maxPageSize, LuceneSliceQueue sliceQueue, int limit, ScoreMode scoreMode) { + public LuceneSourceOperator( + BlockFactory blockFactory, + int maxPageSize, + LuceneSliceQueue sliceQueue, + int limit, + Limiter limiter, + ScoreMode scoreMode + ) { super(blockFactory, maxPageSize, sliceQueue); this.minPageSize = Math.max(1, maxPageSize / 2); this.remainingDocs = limit; + this.limiter = limiter; int estimatedSize = Math.min(limit, maxPageSize); boolean success = false; try { @@ -140,7 +152,7 @@ public void collect(int doc) throws IOException { @Override public boolean isFinished() { - return doneCollecting; + return doneCollecting || limiter.remaining() == 0; } @Override @@ -160,6 +172,7 @@ public Page getCheckedOutput() throws IOException { if (scorer == null) { return null; } + final int remainingDocsStart = remainingDocs = limiter.remaining(); try { scorer.scoreNextRange( leafCollector, @@ -171,28 +184,32 @@ public Page getCheckedOutput() throws IOException { ); } catch (CollectionTerminatedException ex) { // The leaf collector terminated the execution + doneCollecting = true; scorer.markAsDone(); } + final int collectedDocs = remainingDocsStart - remainingDocs; + final int discardedDocs = collectedDocs - limiter.tryAccumulateHits(collectedDocs); Page page = null; - if (currentPagePos >= minPageSize || remainingDocs <= 0 || scorer.isDone()) { - IntBlock shard = null; - IntBlock leaf = null; + if (currentPagePos >= minPageSize || scorer.isDone() || (remainingDocs = limiter.remaining()) == 0) { + IntVector shard = null; + IntVector leaf = null; IntVector docs = null; DoubleVector scores = null; DocBlock docBlock = null; + currentPagePos -= discardedDocs; try { - shard = blockFactory.newConstantIntBlockWith(scorer.shardContext().index(), currentPagePos); - leaf = blockFactory.newConstantIntBlockWith(scorer.leafReaderContext().ord, currentPagePos); - docs = docsBuilder.build(); + shard = blockFactory.newConstantIntVector(scorer.shardContext().index(), currentPagePos); + leaf = blockFactory.newConstantIntVector(scorer.leafReaderContext().ord, currentPagePos); + docs = buildDocsVector(currentPagePos); docsBuilder = blockFactory.newIntVectorBuilder(Math.min(remainingDocs, maxPageSize)); - docBlock = new DocVector(shard.asVector(), leaf.asVector(), docs, true).asBlock(); + docBlock = new DocVector(shard, leaf, docs, true).asBlock(); shard = null; leaf = null; docs = null; if (scoreBuilder == null) { page = new Page(currentPagePos, docBlock); } else { - scores = scoreBuilder.build(); + scores = buildScoresVector(currentPagePos); scoreBuilder = blockFactory.newDoubleVectorBuilder(Math.min(remainingDocs, maxPageSize)); page = new Page(currentPagePos, docBlock, scores.asBlock()); } @@ -209,6 +226,38 @@ public Page getCheckedOutput() throws IOException { } } + private IntVector buildDocsVector(int upToPositions) { + final IntVector docs = docsBuilder.build(); + assert docs.getPositionCount() >= upToPositions : docs.getPositionCount() + " < " + upToPositions; + if (docs.getPositionCount() == upToPositions) { + return docs; + } + try (docs) { + try (var slice = blockFactory.newIntVectorFixedBuilder(upToPositions)) { + for (int i = 0; i < upToPositions; i++) { + slice.appendInt(docs.getInt(i)); + } + return slice.build(); + } + } + } + + private DoubleVector buildScoresVector(int upToPositions) { + final DoubleVector scores = scoreBuilder.build(); + assert scores.getPositionCount() >= upToPositions : scores.getPositionCount() + " < " + upToPositions; + if (scores.getPositionCount() == upToPositions) { + return scores; + } + try (scores) { + try (var slice = blockFactory.newDoubleVectorBuilder(upToPositions)) { + for (int i = 0; i < upToPositions; i++) { + slice.appendDouble(scores.getDouble(i)); + } + return slice.build(); + } + } + } + @Override public void close() { Releasables.close(docsBuilder, scoreBuilder); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 841789e8ada3c..3bccdf75afac3 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -220,9 +220,8 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa positionFieldWork(shard, segment, firstDoc); StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; List rowStrideReaders = new ArrayList<>(fields.length); - ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(blockFactory, docs.count()); LeafReaderContext ctx = ctx(shard, segment); - try { + try (ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(blockFactory, docs.count())) { for (int f = 0; f < fields.length; f++) { FieldWork field = fields[f]; BlockLoader.ColumnAtATimeReader columnAtATime = field.columnAtATime(ctx); @@ -345,27 +344,28 @@ void run() throws IOException { builders[f] = new Block.Builder[shardContexts.size()]; converters[f] = new BlockLoader[shardContexts.size()]; } - ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(blockFactory, docs.getPositionCount()); - int p = forwards[0]; - int shard = shards.getInt(p); - int segment = segments.getInt(p); - int firstDoc = docs.getInt(p); - positionFieldWork(shard, segment, firstDoc); - LeafReaderContext ctx = ctx(shard, segment); - fieldsMoved(ctx, shard); - verifyBuilders(loaderBlockFactory, shard); - read(firstDoc, shard); - for (int i = 1; i < forwards.length; i++) { - p = forwards[i]; - shard = shards.getInt(p); - segment = segments.getInt(p); - boolean changedSegment = positionFieldWorkDocGuarteedAscending(shard, segment); - if (changedSegment) { - ctx = ctx(shard, segment); - fieldsMoved(ctx, shard); - } + try (ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(blockFactory, docs.getPositionCount())) { + int p = forwards[0]; + int shard = shards.getInt(p); + int segment = segments.getInt(p); + int firstDoc = docs.getInt(p); + positionFieldWork(shard, segment, firstDoc); + LeafReaderContext ctx = ctx(shard, segment); + fieldsMoved(ctx, shard); verifyBuilders(loaderBlockFactory, shard); - read(docs.getInt(p), shard); + read(firstDoc, shard); + for (int i = 1; i < forwards.length; i++) { + p = forwards[i]; + shard = shards.getInt(p); + segment = segments.getInt(p); + boolean changedSegment = positionFieldWorkDocGuarteedAscending(shard, segment); + if (changedSegment) { + ctx = ctx(shard, segment); + fieldsMoved(ctx, shard); + } + verifyBuilders(loaderBlockFactory, shard); + read(docs.getInt(p), shard); + } } for (int f = 0; f < target.length; f++) { for (int s = 0; s < shardContexts.size(); s++) { @@ -614,7 +614,7 @@ public String toString() { } } - private static class ComputeBlockLoaderFactory implements BlockLoader.BlockFactory { + private static class ComputeBlockLoaderFactory implements BlockLoader.BlockFactory, Releasable { private final BlockFactory factory; private final int pageSize; private Block nullBlock; @@ -683,12 +683,18 @@ public BlockLoader.Builder nulls(int expectedCount) { public Block constantNulls() { if (nullBlock == null) { nullBlock = factory.newConstantNullBlock(pageSize); - } else { - nullBlock.incRef(); } + nullBlock.incRef(); return nullBlock; } + @Override + public void close() { + if (nullBlock != null) { + nullBlock.close(); + } + } + @Override public BytesRefBlock constantBytes(BytesRef value) { return factory.newConstantBytesRefBlockWith(value, pageSize); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverScheduler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverScheduler.java index 05fe38007a929..9d82f73f3105f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverScheduler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverScheduler.java @@ -7,7 +7,9 @@ package org.elasticsearch.compute.operator; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import java.util.List; import java.util.concurrent.Executor; @@ -21,7 +23,7 @@ */ final class DriverScheduler { private final AtomicReference delayedTask = new AtomicReference<>(); - private final AtomicReference scheduledTask = new AtomicReference<>(); + private final AtomicReference scheduledTask = new AtomicReference<>(); private final AtomicBoolean completing = new AtomicBoolean(); void addOrRunDelayedTask(Runnable task) { @@ -35,22 +37,32 @@ void addOrRunDelayedTask(Runnable task) { } } - void scheduleOrRunTask(Executor executor, Runnable task) { - final Runnable existing = scheduledTask.getAndSet(task); + void scheduleOrRunTask(Executor executor, AbstractRunnable task) { + final AbstractRunnable existing = scheduledTask.getAndSet(task); assert existing == null : existing; final Executor executorToUse = completing.get() ? EsExecutors.DIRECT_EXECUTOR_SERVICE : executor; - executorToUse.execute(() -> { - final Runnable next = scheduledTask.getAndSet(null); - if (next != null) { - assert next == task; - next.run(); + executorToUse.execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + assert e instanceof EsRejectedExecutionException : new AssertionError(e); + if (scheduledTask.getAndUpdate(t -> t == task ? null : t) == task) { + task.onFailure(e); + } + } + + @Override + protected void doRun() { + AbstractRunnable toRun = scheduledTask.getAndSet(null); + if (toRun == task) { + task.run(); + } } }); } void runPendingTasks() { completing.set(true); - for (var taskHolder : List.of(delayedTask, scheduledTask)) { + for (var taskHolder : List.of(scheduledTask, delayedTask)) { final Runnable task = taskHolder.getAndSet(null); if (task != null) { task.run(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java index b669be9192d06..3ef9c420f59ff 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java @@ -22,15 +22,6 @@ import java.util.Objects; public class LimitOperator implements Operator { - /** - * Total number of position that are emitted by this operator. - */ - private final int limit; - - /** - * Remaining number of positions that will be emitted by this operator. - */ - private int limitRemaining; /** * Count of pages that have been processed by this operator. @@ -49,35 +40,49 @@ public class LimitOperator implements Operator { private Page lastInput; + private final Limiter limiter; private boolean finished; - public LimitOperator(int limit) { - this.limit = this.limitRemaining = limit; + public LimitOperator(Limiter limiter) { + this.limiter = limiter; } - public record Factory(int limit) implements OperatorFactory { + public static final class Factory implements OperatorFactory { + private final Limiter limiter; + + public Factory(int limit) { + this.limiter = new Limiter(limit); + } @Override public LimitOperator get(DriverContext driverContext) { - return new LimitOperator(limit); + return new LimitOperator(limiter); } @Override public String describe() { - return "LimitOperator[limit = " + limit + "]"; + return "LimitOperator[limit = " + limiter.limit() + "]"; } } @Override public boolean needsInput() { - return finished == false && lastInput == null; + return finished == false && lastInput == null && limiter.remaining() > 0; } @Override public void addInput(Page page) { assert lastInput == null : "has pending input page"; - lastInput = page; - rowsReceived += page.getPositionCount(); + final int acceptedRows = limiter.tryAccumulateHits(page.getPositionCount()); + if (acceptedRows == 0) { + page.releaseBlocks(); + assert isFinished(); + } else if (acceptedRows < page.getPositionCount()) { + lastInput = truncatePage(page, acceptedRows); + } else { + lastInput = page; + } + rowsReceived += acceptedRows; } @Override @@ -87,7 +92,7 @@ public void finish() { @Override public boolean isFinished() { - return finished && lastInput == null; + return lastInput == null && (finished || limiter.remaining() == 0); } @Override @@ -95,47 +100,38 @@ public Page getOutput() { if (lastInput == null) { return null; } - - Page result; - if (lastInput.getPositionCount() <= limitRemaining) { - result = lastInput; - limitRemaining -= lastInput.getPositionCount(); - } else { - int[] filter = new int[limitRemaining]; - for (int i = 0; i < limitRemaining; i++) { - filter[i] = i; - } - Block[] blocks = new Block[lastInput.getBlockCount()]; - boolean success = false; - try { - for (int b = 0; b < blocks.length; b++) { - blocks[b] = lastInput.getBlock(b).filter(filter); - } - success = true; - } finally { - if (success == false) { - Releasables.closeExpectNoException(lastInput::releaseBlocks, Releasables.wrap(blocks)); - } else { - lastInput.releaseBlocks(); - } - lastInput = null; - } - result = new Page(blocks); - limitRemaining = 0; - } - if (limitRemaining == 0) { - finished = true; - } + final Page result = lastInput; lastInput = null; pagesProcessed++; rowsEmitted += result.getPositionCount(); + return result; + } + private static Page truncatePage(Page page, int upTo) { + int[] filter = new int[upTo]; + for (int i = 0; i < upTo; i++) { + filter[i] = i; + } + final Block[] blocks = new Block[page.getBlockCount()]; + Page result = null; + try { + for (int b = 0; b < blocks.length; b++) { + blocks[b] = page.getBlock(b).filter(filter); + } + result = new Page(blocks); + } finally { + if (result == null) { + Releasables.closeExpectNoException(page::releaseBlocks, Releasables.wrap(blocks)); + } else { + page.releaseBlocks(); + } + } return result; } @Override public Status status() { - return new Status(limit, limitRemaining, pagesProcessed, rowsReceived, rowsEmitted); + return new Status(limiter.limit(), limiter.remaining(), pagesProcessed, rowsReceived, rowsEmitted); } @Override @@ -147,6 +143,8 @@ public void close() { @Override public String toString() { + final int limitRemaining = limiter.remaining(); + final int limit = limiter.limit(); return "LimitOperator[limit = " + limitRemaining + "/" + limit + "]"; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Limiter.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Limiter.java new file mode 100644 index 0000000000000..a74a93eceec40 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Limiter.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A shared limiter used by multiple drivers to collect hits in parallel without exceeding the output limit. + * For example, if the query `FROM test-1,test-2 | LIMIT 100` is run with two drivers, and one driver (e.g., querying `test-1`) + * has collected 60 hits, then the other driver querying `test-2` should collect at most 40 hits. + */ +public class Limiter { + private final int limit; + private final AtomicInteger collected = new AtomicInteger(); + + public static Limiter NO_LIMIT = new Limiter(Integer.MAX_VALUE) { + @Override + public int tryAccumulateHits(int numHits) { + return numHits; + } + + @Override + public int remaining() { + return Integer.MAX_VALUE; + } + }; + + public Limiter(int limit) { + this.limit = limit; + } + + /** + * Returns the remaining number of hits that can be collected. + */ + public int remaining() { + final int remaining = limit - collected.get(); + assert remaining >= 0 : remaining; + return remaining; + } + + /** + * Returns the limit of this limiter. + */ + public int limit() { + return limit; + } + + /** + * Tries to accumulate hits and returns the number of hits that has been accepted. + * + * @param numHits the number of hits to try to accumulate + * @return the accepted number of hits. If the returned number is less than the numHits, + * it means the limit has been reached and the difference can be discarded. + */ + public int tryAccumulateHits(int numHits) { + while (true) { + int curVal = collected.get(); + if (curVal >= limit) { + return 0; + } + final int toAccept = Math.min(limit - curVal, numHits); + if (collected.compareAndSet(curVal, curVal + toAccept)) { + return toAccept; + } + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java index ce400ddbdd6f9..23c98a1df193d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java @@ -47,7 +47,17 @@ void addPage(Page page) { notifyNotEmpty(); } if (noMoreInputs) { - discardPages(); + // O(N) but acceptable because it only occurs with the stop API, and the queue size should be very small. + if (queue.removeIf(p -> p == page)) { + page.releaseBlocks(); + final int size = queueSize.decrementAndGet(); + if (size == maxSize - 1) { + notifyNotFull(); + } + if (size == 0) { + completionFuture.onResponse(null); + } + } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index ac02273a48ee4..dd36a6f455e8b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -366,7 +367,13 @@ private void doFetchPageAsync(boolean allSourcesFinished, ActionListener 0) { // This doesn't fully protect ESQL from OOM, but reduces the likelihood. - blockFactory.breaker().addEstimateBytesAndMaybeBreak(reservedBytes, "fetch page"); + try { + blockFactory.breaker().addEstimateBytesAndMaybeBreak(reservedBytes, "fetch page"); + } catch (Exception e) { + assert e instanceof CircuitBreakingException : new AssertionError(e); + listener.onFailure(e); + return; + } listener = ActionListener.runAfter(listener, () -> blockFactory.breaker().addWithoutBreaking(-reservedBytes)); } transportService.sendChildRequest( diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java index db9a62da5d9ea..68f684cdf9dcd 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java @@ -7,19 +7,16 @@ package org.elasticsearch.compute.operator.exchange; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.FailureCollector; import org.elasticsearch.compute.operator.IsBlockedResult; import org.elasticsearch.core.Releasable; +import org.elasticsearch.tasks.TaskCancelledException; -import java.util.List; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -38,10 +35,9 @@ public final class ExchangeSourceHandler { private final PendingInstances outstandingSinks; private final PendingInstances outstandingSources; - // Collect failures that occur while fetching pages from the remote sink with `failFast=true`. - // The exchange source will stop fetching and abort as soon as any failure is added to this failure collector. - // The final failure collected will be notified to callers via the {@code completionListener}. - private final FailureCollector failure = new FailureCollector(); + // Track if this exchange source should abort. There is no need to track the actual failure since the actual failure + // should be notified via #addRemoteSink(RemoteSink, boolean, Runnable, int, ActionListener). + private volatile boolean aborted = false; private final AtomicInteger nextSinkId = new AtomicInteger(); private final Map remoteSinks = ConcurrentCollections.newConcurrentMap(); @@ -52,35 +48,18 @@ public final class ExchangeSourceHandler { * @param maxBufferSize the maximum size of the exchange buffer. A larger buffer reduces ``pauses`` but uses more memory, * which could otherwise be allocated for other purposes. * @param fetchExecutor the executor used to fetch pages. - * @param completionListener a listener that will be notified when the exchange source handler fails or completes */ - public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor, ActionListener completionListener) { + public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor) { this.buffer = new ExchangeBuffer(maxBufferSize); this.fetchExecutor = fetchExecutor; this.outstandingSinks = new PendingInstances(() -> buffer.finish(false)); - final PendingInstances closingSinks = new PendingInstances(() -> {}); - closingSinks.trackNewInstance(); - this.outstandingSources = new PendingInstances(() -> finishEarly(true, ActionListener.running(closingSinks::finishInstance))); - buffer.addCompletionListener(ActionListener.running(() -> { - final ActionListener listener = ActionListener.assertAtLeastOnce(completionListener); - try (RefCountingRunnable refs = new RefCountingRunnable(() -> { - final Exception e = failure.getFailure(); - if (e != null) { - listener.onFailure(e); - } else { - listener.onResponse(null); - } - })) { - closingSinks.completion.addListener(refs.acquireListener()); - for (PendingInstances pending : List.of(outstandingSinks, outstandingSources)) { - // Create an outstanding instance and then finish to complete the completionListener - // if we haven't registered any instances of exchange sinks or exchange sources before. - pending.trackNewInstance(); - pending.completion.addListener(refs.acquireListener()); - pending.finishInstance(); - } - } - })); + this.outstandingSources = new PendingInstances(() -> finishEarly(true, ActionListener.noop())); + } + + private void checkFailure() { + if (aborted) { + throw new TaskCancelledException("remote sinks failed"); + } } private class ExchangeSourceImpl implements ExchangeSource { @@ -90,13 +69,6 @@ private class ExchangeSourceImpl implements ExchangeSource { outstandingSources.trackNewInstance(); } - private void checkFailure() { - Exception e = failure.getFailure(); - if (e != null) { - throw ExceptionsHelper.convertToRuntime(e); - } - } - @Override public Page pollPage() { checkFailure(); @@ -201,7 +173,7 @@ void fetchPage() { while (loopControl.isRunning()) { loopControl.exiting(); // finish other sinks if one of them failed or source no longer need pages. - boolean toFinishSinks = buffer.noMoreInputs() || failure.hasFailure(); + boolean toFinishSinks = buffer.noMoreInputs() || aborted; remoteSink.fetchPageAsync(toFinishSinks, ActionListener.wrap(resp -> { Page page = resp.takePage(); if (page != null) { @@ -231,7 +203,7 @@ void fetchPage() { void onSinkFailed(Exception e) { if (failFast) { - failure.unwrapAndCollect(e); + aborted = true; } buffer.waitForReading().listener().onResponse(null); // resume the Driver if it is being blocked on reading if (finished == false) { @@ -260,12 +232,12 @@ void onSinkComplete() { * - If {@code false}, failures from this remote sink will not cause the exchange source to abort. * Callers must handle these failures notified via {@code listener}. * - If {@code true}, failures from this remote sink will cause the exchange source to abort. - * Callers can safely ignore failures notified via this listener, as they are collected and - * reported by the exchange source. + * * @param onPageFetched a callback that will be called when a page is fetched from the remote sink * @param instances the number of concurrent ``clients`` that this handler should use to fetch pages. * More clients reduce latency, but add overhead. - * @param listener a listener that will be notified when the sink fails or completes + * @param listener a listener that will be notified when the sink fails or completes. Callers must handle failures notified via + * this listener. * @see ExchangeSinkHandler#fetchPageAsync(boolean, ActionListener) */ public void addRemoteSink( @@ -280,11 +252,17 @@ public void addRemoteSink( final ActionListener sinkListener = ActionListener.assertAtLeastOnce( ActionListener.notifyOnce(ActionListener.runBefore(listener, () -> remoteSinks.remove(sinkId))) ); + final Releasable emptySink = addEmptySink(); fetchExecutor.execute(new AbstractRunnable() { + @Override + public void onAfter() { + emptySink.close(); + } + @Override public void onFailure(Exception e) { if (failFast) { - failure.unwrapAndCollect(e); + aborted = true; } buffer.waitForReading().listener().onResponse(null); // resume the Driver if it is being blocked on reading remoteSink.close(ActionListener.running(() -> sinkListener.onFailure(e))); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java index 78ed096c10b3f..2358643dc089e 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java @@ -409,6 +409,42 @@ public final void testMergeThisBigger() { } } + public final void testMergePastEnd() { + int buckets = 10000; + int bucketSize = between(1, 1000); + int target = between(0, buckets); + List values = randomList(buckets, buckets, this::randomValue); + Collections.sort(values); + try (T sort = build(SortOrder.ASC, bucketSize)) { + // Add a single value to the main sort. + for (int b = 0; b < buckets; b++) { + collect(sort, values.get(b), b); + } + + try (T other = build(SortOrder.ASC, bucketSize)) { + // Add *all* values to the target bucket of the secondary sort. + for (int i = 0; i < values.size(); i++) { + if (i != target) { + collect(other, values.get(i), target); + } + } + + // Merge all buckets pairwise. Most of the secondary ones are empty. + for (int b = 0; b < buckets; b++) { + merge(sort, b, other, b); + } + } + + for (int b = 0; b < buckets; b++) { + if (b == target) { + assertBlock(sort, b, values.subList(0, bucketSize)); + } else { + assertBlock(sort, b, List.of(values.get(b))); + } + } + } + } + protected void assertBlock(T sort, int groupId, List values) { var blockFactory = TestBlockFactory.getNonBreakingInstance(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index b7114bb4e9b54..2322ced00bc25 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -25,10 +25,14 @@ import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.Operator; +import org.elasticsearch.compute.operator.PageConsumerOperator; +import org.elasticsearch.compute.operator.SinkOperator; +import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.compute.test.AnyOperatorTestCase; import org.elasticsearch.compute.test.OperatorTestCase; import org.elasticsearch.compute.test.TestResultPageSinkOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -43,6 +47,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import static org.hamcrest.Matchers.both; @@ -93,7 +98,16 @@ private LuceneSourceOperator.Factory simple(DataPartitioning dataPartitioning, i ShardContext ctx = new MockShardContext(reader, 0); Function queryFunction = c -> new MatchAllDocsQuery(); int maxPageSize = between(10, Math.max(10, numDocs)); - return new LuceneSourceOperator.Factory(List.of(ctx), queryFunction, dataPartitioning, 1, maxPageSize, limit, scoring); + int taskConcurrency = randomIntBetween(1, 4); + return new LuceneSourceOperator.Factory( + List.of(ctx), + queryFunction, + dataPartitioning, + taskConcurrency, + maxPageSize, + limit, + scoring + ); } @Override @@ -117,6 +131,38 @@ public void testShardDataPartitioning() { testSimple(driverContext(), size, limit); } + public void testEarlyTermination() { + int size = between(1_000, 20_000); + int limit = between(0, Integer.MAX_VALUE); + LuceneSourceOperator.Factory factory = simple(randomFrom(DataPartitioning.values()), size, limit, scoring); + int taskConcurrency = factory.taskConcurrency(); + final AtomicInteger receivedRows = new AtomicInteger(); + List drivers = new ArrayList<>(); + for (int i = 0; i < taskConcurrency; i++) { + DriverContext driverContext = driverContext(); + SourceOperator sourceOperator = factory.get(driverContext); + SinkOperator sinkOperator = new PageConsumerOperator(p -> { + receivedRows.addAndGet(p.getPositionCount()); + p.releaseBlocks(); + }); + Driver driver = new Driver( + "driver" + i, + 0, + 0, + driverContext, + () -> "test", + sourceOperator, + List.of(), + sinkOperator, + TimeValue.timeValueNanos(1), + () -> {} + ); + drivers.add(driver); + } + OperatorTestCase.runDriver(drivers); + assertThat(receivedRows.get(), equalTo(Math.min(limit, size))); + } + public void testEmpty() { testSimple(driverContext(), 0, between(10, 10_000)); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java index f017fed16cc96..a5a6333bd846c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java @@ -141,7 +141,7 @@ public void doClose() { if (randomBoolean()) { int limit = between(0, ids.size()); it = ids.subList(0, limit).iterator(); - intermediateOperators.add(new LimitOperator(limit)); + intermediateOperators.add(new LimitOperator(new Limiter(limit))); } else { it = ids.iterator(); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverSchedulerTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverSchedulerTests.java new file mode 100644 index 0000000000000..ec6bf38e557a9 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverSchedulerTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.TestThreadPool; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; + +public class DriverSchedulerTests extends ESTestCase { + + public void testClearPendingTaskOnRejection() { + DriverScheduler scheduler = new DriverScheduler(); + AtomicInteger counter = new AtomicInteger(); + var threadPool = new TestThreadPool( + "test", + new FixedExecutorBuilder(Settings.EMPTY, "test", 1, 2, "test", EsExecutors.TaskTrackingConfig.DEFAULT) + ); + CountDownLatch latch = new CountDownLatch(1); + Executor executor = threadPool.executor("test"); + try { + for (int i = 0; i < 10; i++) { + try { + executor.execute(() -> safeAwait(latch)); + } catch (EsRejectedExecutionException e) { + break; + } + } + scheduler.scheduleOrRunTask(executor, new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + counter.incrementAndGet(); + } + + @Override + protected void doRun() { + counter.incrementAndGet(); + } + }); + scheduler.runPendingTasks(); + assertThat(counter.get(), equalTo(1)); + } finally { + latch.countDown(); + terminate(threadPool); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java index e715b94bc55e5..b6bf0c554856a 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java @@ -331,8 +331,7 @@ public void testResumeOnEarlyFinish() throws Exception { DriverContext driverContext = driverContext(); ThreadPool threadPool = threadPool(); try { - PlainActionFuture sourceFuture = new PlainActionFuture<>(); - var sourceHandler = new ExchangeSourceHandler(between(1, 5), threadPool.executor("esql"), sourceFuture); + var sourceHandler = new ExchangeSourceHandler(between(1, 5), threadPool.executor("esql")); var sinkHandler = new ExchangeSinkHandler(driverContext.blockFactory(), between(1, 5), System::currentTimeMillis); var sourceOperator = new ExchangeSourceOperator(sourceHandler.createExchangeSource()); var sinkOperator = new ExchangeSinkOperator(sinkHandler.createExchangeSink(() -> {}), Function.identity()); @@ -348,7 +347,6 @@ public void testResumeOnEarlyFinish() throws Exception { sinkHandler.fetchPageAsync(true, ActionListener.noop()); future.actionGet(5, TimeUnit.SECONDS); assertThat(driver.status().status(), equalTo(DriverStatus.Status.DONE)); - sourceFuture.actionGet(5, TimeUnit.SECONDS); } finally { terminate(threadPool); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java index 744121a3807c3..e1ca26da035e7 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java @@ -212,11 +212,7 @@ List createDriversForInput(List input, List results, boolean randomIntBetween(2, 10), threadPool.relativeTimeInMillisSupplier() ); - ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler( - randomIntBetween(1, 4), - threadPool.executor(ESQL_TEST_EXECUTOR), - ActionListener.noop() - ); + ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(randomIntBetween(1, 4), threadPool.executor(ESQL_TEST_EXECUTOR)); sourceExchanger.addRemoteSink( sinkExchanger::fetchPageAsync, randomBoolean(), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java index b05be86a164aa..a0f6711051213 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java @@ -13,9 +13,12 @@ import org.elasticsearch.compute.test.OperatorTestCase; import org.elasticsearch.compute.test.RandomBlock; import org.elasticsearch.compute.test.SequenceLongBlockSourceOperator; +import org.elasticsearch.core.TimeValue; import org.hamcrest.Matcher; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.LongStream; import static org.elasticsearch.compute.test.RandomBlock.randomElementType; @@ -126,6 +129,60 @@ public void testBlockPreciselyRemaining() { } } + public void testEarlyTermination() { + int numDrivers = between(1, 4); + final List drivers = new ArrayList<>(); + final int limit = between(1, 10_000); + final LimitOperator.Factory limitFactory = new LimitOperator.Factory(limit); + final AtomicInteger receivedRows = new AtomicInteger(); + for (int i = 0; i < numDrivers; i++) { + DriverContext driverContext = driverContext(); + SourceOperator sourceOperator = new SourceOperator() { + boolean finished = false; + + @Override + public void finish() { + finished = true; + } + + @Override + public boolean isFinished() { + return finished; + } + + @Override + public Page getOutput() { + return new Page(randomBlock(driverContext.blockFactory(), between(1, 100))); + } + + @Override + public void close() { + + } + }; + SinkOperator sinkOperator = new PageConsumerOperator(p -> { + receivedRows.addAndGet(p.getPositionCount()); + p.releaseBlocks(); + }); + drivers.add( + new Driver( + "unset", + 0, + 0, + driverContext, + () -> "test", + sourceOperator, + List.of(limitFactory.get(driverContext)), + sinkOperator, + TimeValue.timeValueMillis(1), + () -> {} + ) + ); + } + runDriver(drivers); + assertThat(receivedRows.get(), equalTo(limit)); + } + Block randomBlock(BlockFactory blockFactory, int size) { if (randomBoolean()) { return blockFactory.newConstantNullBlock(size); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java index bd5b53fb25c8b..7213e0b27aea0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java @@ -66,6 +66,25 @@ public void testDrainPages() throws Exception { blockFactory.ensureAllBlocksAreReleased(); } + public void testOutstandingPages() throws Exception { + ExchangeBuffer buffer = new ExchangeBuffer(randomIntBetween(1000, 10000)); + var blockFactory = blockFactory(); + Page p1 = randomPage(blockFactory); + Page p2 = randomPage(blockFactory); + buffer.addPage(p1); + buffer.addPage(p2); + buffer.finish(false); + buffer.addPage(randomPage(blockFactory)); + assertThat(buffer.size(), equalTo(2)); + assertSame(buffer.pollPage(), p1); + p1.releaseBlocks(); + assertSame(buffer.pollPage(), p2); + p2.releaseBlocks(); + assertNull(buffer.pollPage()); + assertTrue(buffer.isFinished()); + blockFactory.ensureAllBlocksAreReleased(); + } + private static MockBlockFactory blockFactory() { BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java index fffeeac4e4cc2..66e931498ef05 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java @@ -7,15 +7,18 @@ package org.elasticsearch.compute.operator.exchange; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -23,6 +26,7 @@ import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockWritables; import org.elasticsearch.compute.data.IntBlock; @@ -37,6 +41,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancellationService; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.test.transport.StubbableTransport; @@ -69,6 +74,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; public class ExchangeServiceTests extends ESTestCase { @@ -100,16 +106,16 @@ public void testBasic() throws Exception { AtomicInteger pagesAddedToSink = new AtomicInteger(); ExchangeSink sink1 = sinkExchanger.createExchangeSink(pagesAddedToSink::incrementAndGet); ExchangeSink sink2 = sinkExchanger.createExchangeSink(pagesAddedToSink::incrementAndGet); - PlainActionFuture sourceCompletion = new PlainActionFuture<>(); - ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(3, threadPool.executor(ESQL_TEST_EXECUTOR), sourceCompletion); + ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(3, threadPool.executor(ESQL_TEST_EXECUTOR)); ExchangeSource source = sourceExchanger.createExchangeSource(); AtomicInteger pagesAddedToSource = new AtomicInteger(); + PlainActionFuture remoteSinkFuture = new PlainActionFuture<>(); sourceExchanger.addRemoteSink( sinkExchanger::fetchPageAsync, randomBoolean(), pagesAddedToSource::incrementAndGet, 1, - ActionListener.noop() + remoteSinkFuture ); SubscribableListener waitForReading = source.waitForReading().listener(); assertFalse(waitForReading.isDone()); @@ -156,13 +162,12 @@ public void testBasic() throws Exception { sink2.finish(); assertTrue(sink2.isFinished()); assertTrue(source.isFinished()); - assertFalse(sourceCompletion.isDone()); source.finish(); - sourceCompletion.actionGet(10, TimeUnit.SECONDS); ESTestCase.terminate(threadPool); for (Page page : pages) { page.releaseBlocks(); } + safeGet(remoteSinkFuture); } /** @@ -343,47 +348,45 @@ protected void start(Driver driver, ActionListener listener) { public void testConcurrentWithHandlers() { BlockFactory blockFactory = blockFactory(); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var sourceExchanger = new ExchangeSourceHandler( - randomExchangeBuffer(), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); - List sinkHandlers = new ArrayList<>(); - Supplier exchangeSink = () -> { - final ExchangeSinkHandler sinkHandler; - if (sinkHandlers.isEmpty() == false && randomBoolean()) { - sinkHandler = randomFrom(sinkHandlers); - } else { - sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); - sourceExchanger.addRemoteSink( - sinkHandler::fetchPageAsync, - randomBoolean(), - () -> {}, - randomIntBetween(1, 3), - ActionListener.noop() - ); - sinkHandlers.add(sinkHandler); - } - return sinkHandler.createExchangeSink(() -> {}); - }; - final int maxInputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); - final int maxOutputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); - Set actualSeqNos = runConcurrentTest(maxInputSeqNo, maxOutputSeqNo, sourceExchanger::createExchangeSource, exchangeSink); - var expectedSeqNos = IntStream.range(0, Math.min(maxInputSeqNo, maxOutputSeqNo)).boxed().collect(Collectors.toSet()); - assertThat(actualSeqNos, hasSize(expectedSeqNos.size())); - assertThat(actualSeqNos, equalTo(expectedSeqNos)); - sourceCompletionFuture.actionGet(10, TimeUnit.SECONDS); + var sourceExchanger = new ExchangeSourceHandler(randomExchangeBuffer(), threadPool.executor(ESQL_TEST_EXECUTOR)); + PlainActionFuture remoteSinksFuture = new PlainActionFuture<>(); + try (RefCountingListener refs = new RefCountingListener(remoteSinksFuture)) { + List sinkHandlers = new ArrayList<>(); + Supplier exchangeSink = () -> { + final ExchangeSinkHandler sinkHandler; + if (sinkHandlers.isEmpty() == false && randomBoolean()) { + sinkHandler = randomFrom(sinkHandlers); + } else { + sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); + sourceExchanger.addRemoteSink( + sinkHandler::fetchPageAsync, + randomBoolean(), + () -> {}, + randomIntBetween(1, 3), + refs.acquire() + ); + sinkHandlers.add(sinkHandler); + } + return sinkHandler.createExchangeSink(() -> {}); + }; + final int maxInputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); + final int maxOutputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); + Set actualSeqNos = runConcurrentTest( + maxInputSeqNo, + maxOutputSeqNo, + sourceExchanger::createExchangeSource, + exchangeSink + ); + var expectedSeqNos = IntStream.range(0, Math.min(maxInputSeqNo, maxOutputSeqNo)).boxed().collect(Collectors.toSet()); + assertThat(actualSeqNos, hasSize(expectedSeqNos.size())); + assertThat(actualSeqNos, equalTo(expectedSeqNos)); + } + safeGet(remoteSinksFuture); } public void testExchangeSourceContinueOnFailure() { BlockFactory blockFactory = blockFactory(); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var exchangeSourceHandler = new ExchangeSourceHandler( - randomExchangeBuffer(), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); + var exchangeSourceHandler = new ExchangeSourceHandler(randomExchangeBuffer(), threadPool.executor(ESQL_TEST_EXECUTOR)); final int maxInputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); final int maxOutputSeqNo = rarely() ? -1 : randomIntBetween(0, 50_000); Set expectedSeqNos = ConcurrentCollections.newConcurrentSet(); @@ -391,57 +394,65 @@ public void testExchangeSourceContinueOnFailure() { AtomicInteger totalSinks = new AtomicInteger(); AtomicInteger failedSinks = new AtomicInteger(); AtomicInteger completedSinks = new AtomicInteger(); - Supplier exchangeSink = () -> { - var sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); - int failAfter = randomBoolean() ? Integer.MAX_VALUE : randomIntBetween(0, 100); - AtomicInteger fetched = new AtomicInteger(); - int instance = randomIntBetween(1, 3); - totalSinks.incrementAndGet(); - AtomicBoolean sinkFailed = new AtomicBoolean(); - exchangeSourceHandler.addRemoteSink((allSourcesFinished, listener) -> { - if (fetched.incrementAndGet() > failAfter) { - sinkHandler.fetchPageAsync(true, listener.delegateFailure((l, r) -> { - failedRequests.incrementAndGet(); - sinkFailed.set(true); - listener.onFailure(new CircuitBreakingException("simulated", CircuitBreaker.Durability.PERMANENT)); - })); - } else { - sinkHandler.fetchPageAsync(allSourcesFinished, listener.delegateFailure((l, r) -> { - Page page = r.takePage(); - if (page != null) { - IntBlock block = page.getBlock(0); - for (int i = 0; i < block.getPositionCount(); i++) { - int v = block.getInt(i); - if (v < maxOutputSeqNo) { - expectedSeqNos.add(v); + PlainActionFuture remoteSinksFuture = new PlainActionFuture<>(); + try (RefCountingListener refs = new RefCountingListener(remoteSinksFuture)) { + Supplier exchangeSink = () -> { + var sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier()); + int failAfter = randomBoolean() ? Integer.MAX_VALUE : randomIntBetween(0, 100); + AtomicInteger fetched = new AtomicInteger(); + int instance = randomIntBetween(1, 3); + totalSinks.incrementAndGet(); + AtomicBoolean sinkFailed = new AtomicBoolean(); + ActionListener oneSinkListener = refs.acquire(); + exchangeSourceHandler.addRemoteSink((allSourcesFinished, listener) -> { + if (fetched.incrementAndGet() > failAfter) { + sinkHandler.fetchPageAsync(true, listener.delegateFailure((l, r) -> { + failedRequests.incrementAndGet(); + sinkFailed.set(true); + listener.onFailure(new CircuitBreakingException("simulated", CircuitBreaker.Durability.PERMANENT)); + })); + } else { + sinkHandler.fetchPageAsync(allSourcesFinished, listener.delegateFailure((l, r) -> { + Page page = r.takePage(); + if (page != null) { + IntBlock block = page.getBlock(0); + for (int i = 0; i < block.getPositionCount(); i++) { + int v = block.getInt(i); + if (v < maxOutputSeqNo) { + expectedSeqNos.add(v); + } } } - } - l.onResponse(new ExchangeResponse(blockFactory, page, r.finished())); - })); - } - }, false, () -> {}, instance, ActionListener.wrap(r -> { - assertFalse(sinkFailed.get()); - completedSinks.incrementAndGet(); - }, e -> { - assertTrue(sinkFailed.get()); - failedSinks.incrementAndGet(); - })); - return sinkHandler.createExchangeSink(() -> {}); - }; - Set actualSeqNos = runConcurrentTest( - maxInputSeqNo, - maxOutputSeqNo, - exchangeSourceHandler::createExchangeSource, - exchangeSink - ); - assertThat(actualSeqNos, equalTo(expectedSeqNos)); - assertThat(completedSinks.get() + failedSinks.get(), equalTo(totalSinks.get())); - sourceCompletionFuture.actionGet(); + l.onResponse(new ExchangeResponse(blockFactory, page, r.finished())); + })); + } + }, false, () -> {}, instance, ActionListener.wrap(r -> { + assertFalse(sinkFailed.get()); + completedSinks.incrementAndGet(); + oneSinkListener.onResponse(null); + }, e -> { + assertTrue(sinkFailed.get()); + failedSinks.incrementAndGet(); + oneSinkListener.onFailure(e); + })); + return sinkHandler.createExchangeSink(() -> {}); + }; + Set actualSeqNos = runConcurrentTest( + maxInputSeqNo, + maxOutputSeqNo, + exchangeSourceHandler::createExchangeSource, + exchangeSink + ); + assertThat(actualSeqNos, equalTo(expectedSeqNos)); + } if (failedRequests.get() > 0) { + expectThrows(CircuitBreakingException.class, () -> remoteSinksFuture.actionGet(1, TimeUnit.MINUTES)); assertThat(failedSinks.get(), greaterThan(0)); + assertThat(completedSinks.get() + failedSinks.get(), equalTo(totalSinks.get())); } else { + safeGet(remoteSinksFuture); assertThat(failedSinks.get(), equalTo(0)); + assertThat(completedSinks.get(), equalTo(totalSinks.get())); } } @@ -458,7 +469,7 @@ public void testClosingSinks() { assertFalse(sink.waitForWriting().listener().isDone()); PlainActionFuture future = new PlainActionFuture<>(); sinkExchanger.fetchPageAsync(true, future); - ExchangeResponse resp = future.actionGet(); + ExchangeResponse resp = safeGet(future); assertTrue(resp.finished()); assertNull(resp.takePage()); assertTrue(sink.waitForWriting().listener().isDone()); @@ -466,7 +477,7 @@ public void testClosingSinks() { } public void testFinishEarly() throws Exception { - ExchangeSourceHandler sourceHandler = new ExchangeSourceHandler(20, threadPool.generic(), ActionListener.noop()); + ExchangeSourceHandler sourceHandler = new ExchangeSourceHandler(20, threadPool.generic()); Semaphore permits = new Semaphore(between(1, 5)); BlockFactory blockFactory = blockFactory(); Queue pages = ConcurrentCollections.newQueue(); @@ -537,12 +548,7 @@ public void testConcurrentWithTransportActions() { try (exchange0; exchange1; node0; node1) { String exchangeId = "exchange"; Task task = new Task(1, "", "", "", null, Collections.emptyMap()); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var sourceHandler = new ExchangeSourceHandler( - randomExchangeBuffer(), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); + var sourceHandler = new ExchangeSourceHandler(randomExchangeBuffer(), threadPool.executor(ESQL_TEST_EXECUTOR)); ExchangeSinkHandler sinkHandler = exchange1.createSinkHandler(exchangeId, randomExchangeBuffer()); Transport.Connection connection = node0.getConnection(node1.getLocalNode()); sourceHandler.addRemoteSink( @@ -563,7 +569,6 @@ public void testConcurrentWithTransportActions() { var expectedSeqNos = IntStream.range(0, Math.min(maxInputSeqNo, maxOutputSeqNo)).boxed().collect(Collectors.toSet()); assertThat(actualSeqNos, hasSize(expectedSeqNos.size())); assertThat(actualSeqNos, equalTo(expectedSeqNos)); - sourceCompletionFuture.actionGet(10, TimeUnit.SECONDS); } } @@ -613,22 +618,18 @@ public void sendResponse(TransportResponse transportResponse) { try (exchange0; exchange1; node0; node1) { String exchangeId = "exchange"; Task task = new Task(1, "", "", "", null, Collections.emptyMap()); - PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); - var sourceHandler = new ExchangeSourceHandler( - randomIntBetween(1, 128), - threadPool.executor(ESQL_TEST_EXECUTOR), - sourceCompletionFuture - ); + var sourceHandler = new ExchangeSourceHandler(randomIntBetween(1, 128), threadPool.executor(ESQL_TEST_EXECUTOR)); ExchangeSinkHandler sinkHandler = exchange1.createSinkHandler(exchangeId, randomIntBetween(1, 128)); Transport.Connection connection = node0.getConnection(node1.getLocalNode()); + PlainActionFuture remoteSinkFuture = new PlainActionFuture<>(); sourceHandler.addRemoteSink( exchange0.newRemoteSink(task, exchangeId, node0, connection), true, () -> {}, randomIntBetween(1, 5), - ActionListener.noop() + remoteSinkFuture ); - Exception err = expectThrows( + Exception driverException = expectThrows( Exception.class, () -> runConcurrentTest( maxSeqNo, @@ -637,13 +638,36 @@ public void sendResponse(TransportResponse transportResponse) { () -> sinkHandler.createExchangeSink(() -> {}) ) ); - Throwable cause = ExceptionsHelper.unwrap(err, IOException.class); + assertThat(driverException, instanceOf(TaskCancelledException.class)); + var sinkException = expectThrows(Exception.class, remoteSinkFuture::actionGet); + Throwable cause = ExceptionsHelper.unwrap(sinkException, IOException.class); assertNotNull(cause); assertThat(cause.getMessage(), equalTo("page is too large")); PlainActionFuture sinkCompletionFuture = new PlainActionFuture<>(); sinkHandler.addCompletionListener(sinkCompletionFuture); - assertBusy(() -> assertTrue(sinkCompletionFuture.isDone())); - expectThrows(Exception.class, () -> sourceCompletionFuture.actionGet(10, TimeUnit.SECONDS)); + safeGet(sinkCompletionFuture); + } + } + + public void testNoCyclicException() throws Exception { + PlainActionFuture future = new PlainActionFuture<>(); + try (EsqlRefCountingListener refs = new EsqlRefCountingListener(future)) { + var exchangeSourceHandler = new ExchangeSourceHandler(between(10, 100), threadPool.generic()); + int numSinks = between(5, 10); + for (int i = 0; i < numSinks; i++) { + RemoteSink remoteSink = (allSourcesFinished, listener) -> threadPool.schedule( + () -> listener.onFailure(new IOException("simulated")), + TimeValue.timeValueMillis(1), + threadPool.generic() + ); + exchangeSourceHandler.addRemoteSink(remoteSink, randomBoolean(), () -> {}, between(1, 3), refs.acquire()); + } + } + Exception err = expectThrows(Exception.class, () -> future.actionGet(10, TimeUnit.SECONDS)); + assertThat(ExceptionsHelper.unwrap(err, IOException.class).getMessage(), equalTo("simulated")); + try (BytesStreamOutput output = new BytesStreamOutput()) { + // ensure no cyclic exception + ElasticsearchException.writeException(err, output); } } diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/Clusters.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/Clusters.java index 8a55624ed3a6e..654e569535a05 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/Clusters.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/Clusters.java @@ -14,7 +14,7 @@ public class Clusters { public static ElasticsearchCluster mixedVersionCluster() { Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); - return ElasticsearchCluster.local() + var cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .withNode(node -> node.version(oldVersion)) .withNode(node -> node.version(Version.CURRENT)) @@ -22,7 +22,12 @@ public static ElasticsearchCluster mixedVersionCluster() { .withNode(node -> node.version(Version.CURRENT)) .setting("xpack.security.enabled", "false") .setting("xpack.license.self_generated.type", "trial") - .setting("cluster.routing.rebalance.enable", "none") // disable relocation until we have retry in ESQL - .build(); + .setting("cluster.routing.rebalance.enable", "none"); // disable relocation until we have retry in ESQL + + if (oldVersion.before(Version.fromString("8.18.0"))) { + cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper"); + cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService"); + } + return cluster.build(); } } diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index b838d8ae284a4..791f5dacdce64 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -38,7 +38,10 @@ import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.xpack.esql.ccq.Clusters.REMOTE_CLUSTER_NAME; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class MultiClustersIT extends ESRestTestCase { diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java index 708cf74bceee7..5a12fe22b9561 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java @@ -15,9 +15,12 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase; +import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; +import org.hamcrest.Matcher; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -27,6 +30,12 @@ import org.junit.rules.TestRule; import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { @@ -51,6 +60,8 @@ public void setRemoteClient() throws IOException { } } + private boolean isCCSRequest; + @BeforeClass public static void checkVersion() { assumeTrue("skip if version before 8.18", Clusters.localClusterVersion().onOrAfter(Version.V_8_18_0)); @@ -73,13 +84,20 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin @Override protected String from(String... indexName) { - if (randomBoolean()) { + isCCSRequest = randomBoolean(); + if (isCCSRequest) { return "FROM *:" + String.join(",*:", indexName); } else { return "FROM " + String.join(",", indexName); } } + @Override + public Map runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException { + requestObject.includeCCSMetadata(true); + return super.runEsql(requestObject); + } + @After public void wipeRemoteTestData() throws IOException { try { @@ -89,4 +107,35 @@ public void wipeRemoteTestData() throws IOException { assertEquals(404, re.getResponse().getStatusLine().getStatusCode()); } } + + private MapMatcher getClustersMetadataMatcher() { + MapMatcher mapMatcher = matchesMap(); + mapMatcher = mapMatcher.entry("running", 0); + mapMatcher = mapMatcher.entry("total", 1); + mapMatcher = mapMatcher.entry("failed", 0); + mapMatcher = mapMatcher.entry("partial", 0); + mapMatcher = mapMatcher.entry("successful", 1); + mapMatcher = mapMatcher.entry("skipped", 0); + mapMatcher = mapMatcher.entry( + "details", + matchesMap().entry( + Clusters.REMOTE_CLUSTER_NAME, + matchesMap().entry("_shards", matchesMap().extraOk()) + .entry("took", greaterThanOrEqualTo(0)) + .entry("indices", instanceOf(String.class)) + .entry("status", "successful") + ) + ); + return mapMatcher; + } + + @Override + protected void assertQueryResult(Map result, Matcher columnMatcher, Matcher valuesMatcher) { + var matcher = getResultMatcher(result).entry("columns", columnMatcher).entry("values", valuesMatcher); + if (isCCSRequest) { + matcher = matcher.entry("_clusters", getClustersMetadataMatcher()); + } + assertMap(result, matcher); + } + } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/GenerativeIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/GenerativeIT.java index 9b98c29f5c3e3..d322263ce9182 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/GenerativeIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/GenerativeIT.java @@ -15,7 +15,7 @@ import org.elasticsearch.xpack.esql.qa.rest.generative.GenerativeRestTest; import org.junit.ClassRule; -@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102084") +@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/121754") @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class GenerativeIT extends GenerativeRestTest { @ClassRule diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 7a8511c8b0d4a..ae010188ce52b 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -493,26 +493,16 @@ public void testForceSleepsProfile() throws IOException { if (operators.contains("LuceneSourceOperator")) { assertMap(sleeps, matchesMap().entry("counts", Map.of()).entry("first", List.of()).entry("last", List.of())); } else if (operators.contains("ExchangeSourceOperator")) { - if (operators.contains("ExchangeSinkOperator")) { - assertMap(sleeps, matchesMap().entry("counts", matchesMap().entry("exchange empty", greaterThan(0))).extraOk()); - @SuppressWarnings("unchecked") - List> first = (List>) sleeps.get("first"); - for (Map s : first) { - assertMap(s, sleepMatcher); - } - @SuppressWarnings("unchecked") - List> last = (List>) sleeps.get("last"); - for (Map s : last) { - assertMap(s, sleepMatcher); - } - - } else { - assertMap( - sleeps, - matchesMap().entry("counts", matchesMap().entry("exchange empty", 1)) - .entry("first", List.of(sleepMatcher)) - .entry("last", List.of(sleepMatcher)) - ); + assertMap(sleeps, matchesMap().entry("counts", matchesMap().entry("exchange empty", greaterThan(0))).extraOk()); + @SuppressWarnings("unchecked") + List> first = (List>) sleeps.get("first"); + for (Map s : first) { + assertMap(s, sleepMatcher); + } + @SuppressWarnings("unchecked") + List> last = (List>) sleeps.get("last"); + for (Map s : last) { + assertMap(s, sleepMatcher); } } else { fail("unknown signature: " + operators); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index edfda813333df..3e9134a6775d9 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -242,7 +242,7 @@ protected boolean supportsIndexModeLookup() throws IOException { protected final void doTest() throws Throwable { RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values())); - if (testCase.query.toUpperCase(Locale.ROOT).contains("LOOKUP")) { + if (testCase.query.toUpperCase(Locale.ROOT).contains("LOOKUP_\uD83D\uDC14")) { builder.tables(tables()); } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index ad61c52775eb9..1fdc11174ee09 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.AssertWarnings; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.hamcrest.Matcher; import org.junit.After; import org.junit.Assert; @@ -62,7 +63,7 @@ public void testTimestampFilterFromQuery() throws IOException { // filter includes both indices in the result (all columns, all rows) RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -73,7 +74,7 @@ public void testTimestampFilterFromQuery() throws IOException { // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! builder = timestampFilter("gte", "2024-01-01").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -84,7 +85,7 @@ public void testTimestampFilterFromQuery() throws IOException { // filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds // after eliminating the index filter. All columns are returned. builder = timestampFilter("gte", "2025-01-01").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -102,7 +103,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { // filter includes only test1. Columns and rows of test2 are filtered out RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*")); - assertResultMap( + assertQueryResult( runEsql(builder), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -113,7 +114,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*"); Map result = runEsql(builder); - assertResultMap( + assertQueryResult( result, matchesList().item(matchesMap().entry("name", "_index").entry("type", "keyword")) .item(matchesMap().entry("name", "id1").entry("type", "integer")), @@ -138,7 +139,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*" ); Map result = runEsql(builder); - assertResultMap( + assertQueryResult( result, matchesList().item(matchesMap().entry("name", "_index").entry("type", "keyword")) .item(matchesMap().entry("name", "id1").entry("type", "integer")) @@ -298,4 +299,9 @@ protected void indexTimestampDataForClient(RestClient client, int docs, String i Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); } } + + protected void assertQueryResult(Map result, Matcher columnMatcher, Matcher valuesMatcher) { + assertResultMap(result, columnMatcher, valuesMatcher); + } + } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 93a5e005f6a27..13cfbe32af033 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -84,9 +84,13 @@ public abstract class RestEsqlTestCase extends ESRestTestCase { private static final String MAPPING_ALL_TYPES; + private static final String MAPPING_ALL_TYPES_LOOKUP; + static { String properties = EsqlTestUtils.loadUtf8TextFile("/mapping-all-types.json"); MAPPING_ALL_TYPES = "{\"mappings\": " + properties + "}"; + String settings = "{\"settings\" : {\"mode\" : \"lookup\"}"; + MAPPING_ALL_TYPES_LOOKUP = settings + ", " + "\"mappings\": " + properties + "}"; } private static final String DOCUMENT_TEMPLATE = """ @@ -661,10 +665,6 @@ public void testErrorMessageForArrayValuesInParams() throws IOException { } public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOException { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); bulkLoadTestData(10); // positive var query = requestObjectBuilder().query( @@ -774,6 +774,59 @@ public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOExcepti } } + public void testErrorMessageForMissingParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql(requestObjectBuilder().query("from idx | where x == ?n1").params("[]")) + ); + assertThat( + EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""), + containsString("line 1:23: Unknown query parameter [n1]") + ); + + re = expectThrows( + ResponseException.class, + () -> runEsql(requestObjectBuilder().query("from idx | where x == ?n1 and y == ?n2").params("[{\"n\" : \"v\"}]")) + ); + assertThat(EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""), containsString(""" + line 1:23: Unknown query parameter [n1], did you mean [n]?; line 1:36: Unknown query parameter [n2], did you mean [n]?""")); + + re = expectThrows( + ResponseException.class, + () -> runEsql(requestObjectBuilder().query("from idx | where x == ?n1 and y == ?n2").params("[{\"n1\" : \"v1\"}]")) + ); + assertThat( + EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""), + containsString("line 1:36: Unknown query parameter [n2], did you mean [n1]") + ); + } + + public void testMultipleBatchesWithLookupJoin() throws IOException { + assumeTrue( + "Makes numberOfChannels consistent with layout map for join with multiple batches", + EsqlCapabilities.Cap.MAKE_NUMBER_OF_CHANNELS_CONSISTENT_WITH_LAYOUT.isEnabled() + ); + // Create more than 10 indices to trigger multiple batches of data node execution. + // The sort field should be missing on some indices to reproduce NullPointerException caused by duplicated items in layout + for (int i = 1; i <= 20; i++) { + createIndex("idx" + i, randomBoolean(), "\"mappings\": {\"properties\" : {\"a\" : {\"type\" : \"keyword\"}}}"); + } + bulkLoadTestDataLookupMode(10); + // lookup join with and without sort + for (String sort : List.of("", "| sort integer")) { + var query = requestObjectBuilder().query(format(null, "from * | lookup join {} on integer {}", testIndexName(), sort)); + Map result = runEsql(query); + var columns = as(result.get("columns"), List.class); + assertEquals(21, columns.size()); + var values = as(result.get("values"), List.class); + assertEquals(10, values.size()); + } + // clean up + for (int i = 1; i <= 20; i++) { + assertThat(deleteIndex("idx" + i).isAcknowledged(), is(true)); + } + } + public void testErrorMessageForLiteralDateMathOverflow() throws IOException { List dateMathOverflowExpressions = List.of( "2147483647 day + 1 day", @@ -1355,13 +1408,22 @@ private static void bulkLoadTestData(int count) throws IOException { bulkLoadTestData(count, 0, true, RestEsqlTestCase::createDocument); } + private static void bulkLoadTestDataLookupMode(int count) throws IOException { + createIndex(testIndexName(), true); + bulkLoadTestData(count, 0, false, RestEsqlTestCase::createDocument); + } + + private static void createIndex(String indexName, boolean lookupMode) throws IOException { + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity(lookupMode ? MAPPING_ALL_TYPES_LOOKUP : MAPPING_ALL_TYPES); + assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode()); + } + private static void bulkLoadTestData(int count, int firstIndex, boolean createIndex, IntFunction createDocument) throws IOException { Request request; if (createIndex) { - request = new Request("PUT", "/" + testIndexName()); - request.setJsonEntity(MAPPING_ALL_TYPES); - assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode()); + createIndex(testIndexName(), false); } if (count > 0) { @@ -1436,6 +1498,13 @@ private static String repeatValueAsMV(Object value) { return "[" + value + ", " + value + "]"; } + private static void createIndex(String indexName, boolean lookupMode, String mapping) throws IOException { + Request request = new Request("PUT", "/" + indexName); + String settings = "\"settings\" : {\"mode\" : \"lookup\"}, "; + request.setJsonEntity("{" + (lookupMode ? settings : "") + mapping + "}"); + assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode()); + } + public static RequestObjectBuilder requestObjectBuilder() throws IOException { return new RequestObjectBuilder(); } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java index 844ccd3802bf1..5bf13d2d9c762 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java @@ -28,10 +28,13 @@ public record Column(String name, String type) {} public record QueryExecuted(String query, int depth, List outputSchema, Exception exception) {} public static String sourceCommand(List availabeIndices) { - return switch (randomIntBetween(0, 2)) { + return switch (randomIntBetween(0, 1)) { case 0 -> from(availabeIndices); - case 1 -> metaFunctions(); - default -> row(); + // case 1 -> metaFunctions(); + default -> from(availabeIndices); + // TODO re-enable ROW. + // now it crashes nodes in some cases: exiting java.lang.AssertionError: estimated row size [0] wasn't set + // default -> row(); }; } @@ -41,8 +44,12 @@ public static String sourceCommand(List availabeIndices) { * @param policies * @return a new command that can process it as input */ - public static String pipeCommand(List previousOutput, List policies) { - return switch (randomIntBetween(0, 11)) { + public static String pipeCommand( + List previousOutput, + List policies, + List lookupIndices + ) { + return switch (randomIntBetween(0, 12)) { case 0 -> dissect(previousOutput); case 1 -> drop(previousOutput); case 2 -> enrich(previousOutput, policies); @@ -54,10 +61,26 @@ public static String pipeCommand(List previousOutput, List rename(previousOutput); case 9 -> sort(previousOutput); case 10 -> stats(previousOutput); + case 11 -> join(previousOutput, lookupIndices); default -> where(previousOutput); }; } + private static String join(List previousOutput, List lookupIndices) { + + GenerativeRestTest.LookupIdx lookupIdx = randomFrom(lookupIndices); + String lookupIdxName = lookupIdx.idxName(); + String idxKey = lookupIdx.key(); + String keyType = lookupIdx.keyType(); + + var candidateKeys = previousOutput.stream().filter(x -> x.type.equals(keyType)).toList(); + if (candidateKeys.isEmpty()) { + return ""; + } + Column key = randomFrom(candidateKeys); + return "| rename " + key.name + " as " + idxKey + " | lookup join " + lookupIdxName + " on " + idxKey; + } + private static String where(List previousOutput) { // TODO more complex conditions StringBuilder result = new StringBuilder(" | where "); @@ -191,7 +214,53 @@ private static String keep(List previousOutput) { } private static String randomName(List previousOutput) { - return previousOutput.get(randomIntBetween(0, previousOutput.size() - 1)).name(); + // we need to exclude + // https://github.com/elastic/elasticsearch/issues/121741 + return randomFrom(previousOutput.stream().filter(x -> x.name().equals("") == false).toList()).name(); + } + + private static String randomGroupableName(List previousOutput) { + // we need to exclude + // https://github.com/elastic/elasticsearch/issues/121741 + var candidates = previousOutput.stream() + .filter(EsqlQueryGenerator::groupable) + .filter(x -> x.name().equals("") == false) + .toList(); + if (candidates.isEmpty()) { + return null; + } + return randomFrom(candidates).name(); + } + + private static boolean groupable(Column col) { + return col.type.equals("keyword") + || col.type.equals("text") + || col.type.equals("long") + || col.type.equals("integer") + || col.type.equals("ip") + || col.type.equals("version"); + } + + private static String randomSortableName(List previousOutput) { + // we need to exclude + // https://github.com/elastic/elasticsearch/issues/121741 + var candidates = previousOutput.stream() + .filter(EsqlQueryGenerator::sortable) + .filter(x -> x.name().equals("") == false) + .toList(); + if (candidates.isEmpty()) { + return null; + } + return randomFrom(candidates).name(); + } + + private static boolean sortable(Column col) { + return col.type.equals("keyword") + || col.type.equals("text") + || col.type.equals("long") + || col.type.equals("integer") + || col.type.equals("ip") + || col.type.equals("version"); } private static String rename(List previousOutput) { @@ -199,7 +268,12 @@ private static String rename(List previousOutput) { List proj = new ArrayList<>(); List names = new ArrayList<>(previousOutput.stream().map(Column::name).collect(Collectors.toList())); for (int i = 0; i < n; i++) { - String name = names.remove(randomIntBetween(0, names.size() - 1)); + var colN = randomIntBetween(0, names.size() - 1); + if (previousOutput.get(colN).type().endsWith("_range")) { + // ranges are not fully supported yet + continue; + } + String name = names.remove(colN); String newName; if (names.isEmpty() || randomBoolean()) { newName = randomAlphaOfLength(5); @@ -209,6 +283,9 @@ private static String rename(List previousOutput) { names.add(newName); proj.add(name + " AS " + newName); } + if (proj.isEmpty()) { + return ""; + } return " | rename " + proj.stream().collect(Collectors.joining(", ")); } @@ -227,7 +304,7 @@ private static String drop(List previousOutput) { name = "*" + name.substring(randomIntBetween(1, name.length() - 1)); } } - proj.add(name); + proj.add(name.contains("*") ? name : "`" + name + "`"); } return " | drop " + proj.stream().collect(Collectors.joining(", ")); } @@ -236,7 +313,11 @@ private static String sort(List previousOutput) { int n = randomIntBetween(1, previousOutput.size()); Set proj = new HashSet<>(); for (int i = 0; i < n; i++) { - proj.add(randomName(previousOutput)); + String col = randomSortableName(previousOutput); + if (col == null) { + return "";// no sortable columns + } + proj.add(col); } return " | sort " + proj.stream() @@ -295,9 +376,10 @@ private static String stats(List previousOutput) { cmd.append(expression); } if (randomBoolean()) { - cmd.append(" by "); - - cmd.append(randomName(nonNull)); + var col = randomGroupableName(nonNull); + if (col != null) { + cmd.append(" by " + col); + } } return cmd.toString(); } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 588d5870d89ec..a841c2fc99958 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -35,7 +35,18 @@ public abstract class GenerativeRestTest extends ESRestTestCase { public static final Set ALLOWED_ERRORS = Set.of( "Reference \\[.*\\] is ambiguous", "Cannot use field \\[.*\\] due to ambiguities", - "cannot sort on .*" + "cannot sort on .*", + "argument of \\[count_distinct\\(.*\\)\\] must", + "Cannot use field \\[.*\\] with unsupported type \\[.*_range\\]", + // warnings + "Field '.*' shadowed by field at line .*", + "evaluation of \\[.*\\] failed, treating result as null", // TODO investigate? + // Awaiting fixes + "estimated row size \\[0\\] wasn't set", // https://github.com/elastic/elasticsearch/issues/121739 + "unknown physical plan node \\[OrderExec\\]", // https://github.com/elastic/elasticsearch/issues/120817 + "Unknown column \\[\\]", // https://github.com/elastic/elasticsearch/issues/121741 + // + "The incoming YAML document exceeds the limit:" // still to investigate, but it seems to be specific to the test framework ); public static final Set ALLOWED_ERROR_PATTERNS = ALLOWED_ERRORS.stream() @@ -64,6 +75,7 @@ public static void wipeTestData() throws IOException { public void test() { List indices = availableIndices(); + List lookupIndices = lookupIndices(); List policies = availableEnrichPolicies(); for (int i = 0; i < ITERATIONS; i++) { String command = EsqlQueryGenerator.sourceCommand(indices); @@ -76,7 +88,7 @@ public void test() { if (result.outputSchema().isEmpty()) { break; } - command = EsqlQueryGenerator.pipeCommand(result.outputSchema(), policies); + command = EsqlQueryGenerator.pipeCommand(result.outputSchema(), policies, lookupIndices); result = execute(result.query() + command, result.depth() + 1); if (result.exception() != null) { checkException(result); @@ -102,6 +114,9 @@ private EsqlQueryGenerator.QueryExecuted execute(String command, int depth) { return new EsqlQueryGenerator.QueryExecuted(command, depth, outputSchema, null); } catch (Exception e) { return new EsqlQueryGenerator.QueryExecuted(command, depth, null, e); + } catch (AssertionError ae) { + // this is for ensureNoWarnings() + return new EsqlQueryGenerator.QueryExecuted(command, depth, null, new RuntimeException(ae.getMessage())); } } @@ -116,7 +131,23 @@ private List outputSchema(Map a) { } private List availableIndices() { - return new ArrayList<>(CSV_DATASET_MAP.keySet()); + return new ArrayList<>( + CSV_DATASET_MAP.entrySet() + .stream() + .filter(x -> x.getValue().requiresInferenceEndpoint() == false) + .map(Map.Entry::getKey) + .toList() + ); + } + + record LookupIdx(String idxName, String key, String keyType) {} + + private List lookupIndices() { + List result = new ArrayList<>(); + // we don't have key info from the dataset loader, let's hardcode it for now + result.add(new LookupIdx("languages_lookup", "language_code", "integer")); + result.add(new LookupIdx("message_types_lookup", "message", "keyword")); + return result; } List availableEnrichPolicies() { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index 9177fcbcd2afb..8c186484b7361 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -281,3 +281,111 @@ languages:integer| emp_no:integer|eval:keyword null |10020 |languages is null null |10021 |languages is null ; + +caseWithMixedNumericValue +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", 1.0, gender == "M", 2, 3.0) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 2.0 +10006 | F | 1.0 +10007 | F | 1.0 +10008 | M | 2.0 +10009 | F | 1.0 +10010 | null | 3.0 +; + +caseWithMixedNumericValueWithNull +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", 1.0, gender == "M", 2, null) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 2.0 +10006 | F | 1.0 +10007 | F | 1.0 +10008 | M | 2.0 +10009 | F | 1.0 +10010 | null | null +; + +caseWithMixedNumericField +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", height, gender == "M", salary, languages) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | 1.56 +10007 | F | 1.7 +10008 | M | 43906.0 +10009 | F | 1.85 +10010 | null | 4.0 +; + +caseWithMixedNumericFieldWithNull +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", height, gender == "M", salary, null) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | 1.56 +10007 | F | 1.7 +10008 | M | 43906.0 +10009 | F | 1.85 +10010 | null | null +; + +caseWithMixedNumericFieldWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", salary_change, gender == "M", salary, languages) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | -3.9 +10007 | F | [-7.06, 0.57, 1.99] +10008 | M | 43906.0 +10009 | F | null +10010 | null | 4.0 +; + +caseWithMixedNumericFieldWithNullWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", salary_change, gender == "M", salary, null) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | -3.9 +10007 | F | [-7.06, 0.57, 1.99] +10008 | M | 43906.0 +10009 | F | null +10010 | null | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv index bd5fe7fad3a4e..f79e44ab67ca3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv @@ -1,4 +1,4 @@ -_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text,st_logs:semantic_text -1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==,"2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553" -2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=,"2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42" -3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,,"2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" +_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text,st_logs:semantic_text,language_name:keyword +1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==,"2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553",English +2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=,"2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42",French +3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,,"2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42",Spanish diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index e48f1e9d0b9ae..6ca3fb75f22b5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -216,6 +216,23 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 ; +date nanos greater than, no mv min +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos > TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos > TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948000000Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +; + implicit casting to nanos, date only required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -236,6 +253,29 @@ millis:date | nanos:date_nanos 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z ; +implicit casting to nanos, date only, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos > "2023-10-23" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +; + implicit casting to nanos, date only, equality test required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -248,6 +288,21 @@ FROM date_nanos millis:date | nanos:date_nanos ; +implicit casting to nanos, date only, equality test, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos == "2023-10-23" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +; + implicit casting to nanos, date plus time to seconds required_capability: date_nanos_type @@ -269,6 +324,29 @@ millis:date | nanos:date_nanos 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z ; +implicit casting to nanos, date plus time to seconds, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos > "2023-10-23T00:00:00" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23T00:00:00\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +; + implicit casting to nanos, date plus time to seconds, equality test required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -281,6 +359,21 @@ FROM date_nanos millis:date | nanos:date_nanos ; +implicit casting to nanos, date plus time to seconds, equality test, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos == "2023-10-23T12:27:28" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23T12:27:28\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +; + implicit casting to nanos, date plus time to millis required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -301,6 +394,29 @@ millis:date | nanos:date_nanos 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z ; +implicit casting to nanos, date plus time to millis, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos > "2023-10-23T00:00:00.000" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23T00:00:00\.000\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +; + implicit casting to nanos, date plus time to millis, equality test required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -314,6 +430,22 @@ millis:date | nanos:date_nanos 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z ; +implicit casting to nanos, date plus time to millis, equality test, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos == "2023-10-23T12:27:28.948" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23T12:27:28\.948\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z +; + implicit casting to nanos, date plus time to nanos required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -334,6 +466,29 @@ millis:date | nanos:date_nanos 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z ; +implicit casting to nanos, date plus time to nanos, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos > "2023-10-23T00:00:00.000000000" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos > \\\"2023-10-23T00:00:00\.000000000\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +; + implicit casting to nanos, date plus time to nanos, equality test required_capability: date_nanos_type required_capability: date_nanos_implicit_casting @@ -347,6 +502,22 @@ millis:date | nanos:date_nanos 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z ; +implicit casting to nanos, date plus time to nanos, equality test, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos == "2023-10-23T12:27:28.948000000" +| SORT nanos DESC +| KEEP millis, nanos; +warningRegex:Line 2:9: evaluation of \[nanos == \\\"2023-10-23T12:27:28\.948000000\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z +; + date nanos greater than millis required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis @@ -361,6 +532,23 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 ; +date nanos greater than millis, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948Z") | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +; + date nanos greater than or equal required_capability: to_date_nanos required_capability: date_nanos_binary_comparison @@ -376,6 +564,24 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +date nanos greater than or equal, no mv min +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos >= TO_DATE_NANOS("2023-10-23T12:27:28.948Z") | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos >= TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + date nanos greater than or equal millis required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis @@ -391,6 +597,23 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +date nanos greater than or equal millis, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos >= TO_DATETIME("2023-10-23T12:27:28.948Z") | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos >= TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; date nanos less than required_capability: to_date_nanos @@ -403,12 +626,39 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +date nanos less than, no mv min +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos < TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos < TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + date nanos less than millis required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; +millis:date | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; +date nanos less than millis, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos < TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos < TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 @@ -426,6 +676,21 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +date nanos less than equal, no mv min +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos <= TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos <= TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + date nanos less than equal millis required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis @@ -438,6 +703,20 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +date nanos less than equal millis, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos <= TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos <= TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; date nanos equals required_capability: to_date_nanos @@ -449,6 +728,19 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +date nanos equals, no mv min +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos == TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z"); +warningRegex:Line 1:25: evaluation of \[nanos == TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948000000Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + date nanos equals millis exact match required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis @@ -459,6 +751,19 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +date nanos equals millis exact match, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos == TO_DATETIME("2023-10-23T12:27:28.948Z"); +warningRegex:Line 1:25: evaluation of \[nanos == TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + date nanos equals millis without exact match required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis @@ -468,6 +773,18 @@ FROM date_nanos | WHERE MV_MIN(nanos) == TO_DATETIME("2023-10-23T13:33:34.937"); millis:date | nanos:date_nanos | num:long ; +date nanos equals millis without exact match, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos == TO_DATETIME("2023-10-23T13:33:34.937"); +warningRegex:Line 1:25: evaluation of \[nanos == TO_DATETIME\(\\\"2023-10-23T13:33:34\.937\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +; + date nanos not equals required_capability: to_date_nanos required_capability: date_nanos_binary_comparison @@ -484,6 +801,25 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +date nanos not equals, no mv min +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos != TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos != TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948000000Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + date nanos not equals millis required_capability: date_nanos_type required_capability: date_nanos_compare_to_millis @@ -500,6 +836,25 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +date nanos not equals millis, no mv min +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos | WHERE nanos != TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; +warningRegex:Line 1:25: evaluation of \[nanos != TO_DATETIME\(\\\"2023-10-23T12:27:28\.948Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 1:25: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + Date nanos date extract required_capability: date_nanos_date_extract @@ -522,6 +877,34 @@ nn:date_nanos | year:long | ns:long 2023-03-23T12:15:03.360103847Z | 2023 | 360103847 ; +Date nanos date extract, no mv max +required_capability: date_nanos_date_extract + +FROM date_nanos +| EVAL nn = nanos +| EVAL year = DATE_EXTRACT("year", nn), ns = DATE_EXTRACT("nano_of_second", nn) +| KEEP nn, year, ns +| SORT nn DESC; +# NB: sorting of the MV results isn't deterministic, so we ignore ordering here +ignoreOrder:true +warning:Line 3:15: evaluation of [DATE_EXTRACT(\"year\", nn)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:15: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 3:46: evaluation of [DATE_EXTRACT(\"nano_of_second\", nn)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:46: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nn:date_nanos | year:long | ns:long +2023-10-23T13:55:01.543123456Z | 2023 | 543123456 +2023-10-23T13:53:55.832987654Z | 2023 | 832987654 +2023-10-23T13:52:55.015787878Z | 2023 | 015787878 +2023-10-23T13:51:54.732102837Z | 2023 | 732102837 +2023-10-23T13:33:34.937193000Z | 2023 | 937193000 +2023-10-23T12:27:28.948000000Z | 2023 | 948000000 +2023-10-23T12:15:03.360103847Z | 2023 | 360103847 +2023-10-23T12:15:03.360103847Z | 2023 | 360103847 +[2023-01-23T13:55:01.543123456Z, 2023-02-23T13:33:34.937193Z, 2023-03-23T12:15:03.360103847Z] | null | null +[2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z, 2023-03-23T12:15:03.360103847Z] | null | null +; + date nanos to long, index version required_capability: to_date_nanos @@ -707,7 +1090,8 @@ required_capability: date_nanos_bucket FROM date_nanos | WHERE millis > "2020-01-01" -| STATS ct = count(*) BY yr = BUCKET(nanos, 1 year); +| STATS ct = count(*) BY yr = BUCKET(nanos, 1 year) +| SORT yr DESC; ct:long | yr:date_nanos 8 | 2023-01-01T00:00:00.000000000Z @@ -719,7 +1103,8 @@ required_capability: date_nanos_bucket FROM date_nanos | WHERE millis > "2020-01-01" -| STATS ct = count(*) BY yr = BUCKET(nanos, 5, "1999-01-01", NOW()); +| STATS ct = count(*) BY yr = BUCKET(nanos, 5, "1999-01-01", NOW()) +| SORT yr DESC; ct:long | yr:date_nanos 8 | 2023-01-01T00:00:00.000000000Z @@ -731,7 +1116,8 @@ required_capability: date_nanos_bucket FROM date_nanos | WHERE millis > "2020-01-01" -| STATS ct = count(*) BY mo = BUCKET(nanos, 1 month); +| STATS ct = count(*) BY mo = BUCKET(nanos, 1 month) +| SORT mo DESC; ct:long | mo:date_nanos 8 | 2023-10-01T00:00:00.000000000Z @@ -743,7 +1129,8 @@ required_capability: date_nanos_bucket FROM date_nanos | WHERE millis > "2020-01-01" -| STATS ct = count(*) BY mo = BUCKET(nanos, 20, "2023-01-01", "2023-12-31"); +| STATS ct = count(*) BY mo = BUCKET(nanos, 20, "2023-01-01", "2023-12-31") +| SORT mo DESC; ct:long | mo:date_nanos 8 | 2023-10-01T00:00:00.000000000Z @@ -755,11 +1142,13 @@ required_capability: date_nanos_bucket FROM date_nanos | WHERE millis > "2020-01-01" -| STATS ct = count(*) BY mo = BUCKET(nanos, 55, "2023-01-01", "2023-12-31"); +| STATS ct = count(*) BY mo = BUCKET(nanos, 55, "2023-01-01", "2023-12-31") +| SORT mo DESC; ct:long | mo:date_nanos 8 | 2023-10-23T00:00:00.000000000Z ; + Bucket Date nanos by 10 minutes required_capability: date_trunc_date_nanos required_capability: date_nanos_bucket @@ -767,7 +1156,8 @@ required_capability: string_literal_auto_casting FROM date_nanos | WHERE millis > "2020-01-01" -| STATS ct = count(*) BY mn = BUCKET(nanos, 10 minutes); +| STATS ct = count(*) BY mn = BUCKET(nanos, 10 minutes) +| SORT mn DESC; ct:long | mn:date_nanos 4 | 2023-10-23T13:50:00.000000000Z @@ -1190,6 +1580,22 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +Date Nanos IN constant date nanos, no mv first +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z")); +ignoreOrder:true +warningRegex:Line 2:9: evaluation of \[nanos IN \(TO_DATE_NANOS\(\\\"2023-10-23T13:55:01\.543123456Z\\\"\), TO_DATE_NANOS\(\\\"2023-10-23T12:27:28\.948Z\\\"\), TO_DATE_NANOS\(\\\"2017-10-23T13:53:55\.832987654Z\\\"\)\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + Date Nanos IN constant date nanos, implicit casting required_capability: date_nanos_in_operator required_capability: to_date_nanos @@ -1204,6 +1610,23 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +Date Nanos IN constant date nanos, implicit casting, no mv function +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE nanos IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z"); +ignoreOrder:true +warningRegex:Line 2:9: evaluation of \[nanos IN \(\\\"2023-10-23T13:55:01\.543123456Z\\\", \\\"2023-10-23T12:27:28\.948Z\\\", \\\"2017-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + Date Nanos IN date nanos field, implicit casting required_capability: date_nanos_in_operator required_capability: to_date_nanos @@ -1216,6 +1639,23 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 ; +Date Nanos IN date nanos field, implicit casting, no mv function +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE "2023-10-23T13:55:01.543123456Z" IN (nanos); +ignoreOrder:true +warningRegex:Line 2:9: evaluation of \[\\\"2023-10-23T13:55:01\.543123456Z\\\" IN \(nanos\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +; + +# TODO: should this one work without the mv function? doesn't seem to. Date nanos IN millisecond date field required_capability: date_nanos_in_operator required_capability: to_date_nanos @@ -1288,3 +1728,135 @@ diff_sec:integer | diff_sec_m:integer | n:date_nanos -18489600 | -18489599 | 2023-03-23T12:15:03.360103847Z -18489600 | -18489599 | 2023-03-23T12:15:03.360103847Z ; + +Regression out of bounds in where clause +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos >= to_datenanos("2010-03-20T15:08:25.608Z") AND nanos <= to_datenanos("2026-03-20T15:08:25.608Z") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:63: evaluation of \[nanos <= to_datenanos\(\\\"2026-03-20T15:08:25\.608Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:63: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos >= to_datenanos\(\\\"2010-03-20T15:08:25\.608Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193Z +2023-10-23T12:27:28.948Z +2023-10-23T12:15:03.360103847Z +2023-10-23T12:15:03.360103847Z +; + +Range Without Included Endpoints +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datenanos("2023-10-23T13:53:55.832987654Z") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:67: evaluation of \[nanos < to_datenanos\(\\\"2023-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; + +Range Without Included Endpoints with implicit casting +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > "2023-10-23T12:15:03.360103847" AND nanos < "2023-10-23T13:53:55.832987654Z" +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:53: evaluation of \[nanos < \\\"2023-10-23T13:53:55\.832987654Z\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:53: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > \\\"2023-10-23T12:15:03\.360103847\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; + +Range With Now date math +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948") AND nanos < now() - 1d +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:60: evaluation of \[nanos < now\(\) - 1d\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:60: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +; + +Mixed Nanos Millis Range Compare +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug +required_capability: fix_date_nanos_mixed_range_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datetime("2023-10-23T13:53:55.832") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; + +Mixed Nanos Millis Range Compare, millis field +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug +required_capability: fix_date_nanos_mixed_range_pushdown_bug + +FROM date_nanos +| WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec index 3c38bd190b0b1..0a7cb5cbceaf8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec @@ -644,3 +644,21 @@ FROM airports abbrev:k | city_name:k | city_location:geo_point | country:k | location:geo_point | name:text | region:text | boundary_wkt_length:i IDR | Indore | POINT(75.8472 22.7167) | India | POINT(75.8092915005895 22.727749187571) | Devi Ahilyabai Holkar Int'l | Indore City | 231 ; + +// Regression test for https://github.com/elastic/elasticsearch/issues/126030 +// We had wrong layouts from ReplaceMissingFieldsWithNull in case of indices that had relevant fields for the query, +// but were **missing the field we enrich on**. +fieldsInOtherIndicesBug +required_capability: enrich_load +required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout + +from * +| keep author.keyword, book_no, scalerank, street, bytes_in, @timestamp, abbrev, city_location, distance, description, birth_date, language_code, intersects, client_ip, event_duration, version +| enrich languages_policy on author.keyword +| sort book_no +| limit 1 +; + +author.keyword:keyword|book_no:keyword|scalerank:integer|street:keyword|bytes_in:ul|@timestamp:unsupported|abbrev:keyword|city_location:geo_point|distance:double|description:unsupported|birth_date:date|language_code:integer|intersects:boolean|client_ip:unsupported|event_duration:long|version:version|language_name:keyword +Fyodor Dostoevsky |1211 |null |null |null |null |null |null |null |null |null |null |null |null |null |null |null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index d4a98fdc70a9a..ee9d25c7d4474 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1346,3 +1346,159 @@ language_code:integer | language_name:keyword | country:text 1 | English | United States of America 1 | English | null ; + + +sortBeforeAndAfterJoin +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort + +FROM employees +| sort first_name +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE emp_no >= 10091 AND emp_no < 10094 +| SORT emp_no +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10092 | 1 | English +10093 | 3 | Spanish +; + + + +sortBeforeAndAfterMultipleJoinAndMvExpand +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort + +FROM employees +| sort first_name +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE emp_no >= 10091 AND emp_no < 10094 +| SORT language_name +| MV_EXPAND first_name +| SORT first_name +| MV_EXPAND last_name +| SORT last_name +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10092 | 1 | English +10093 | 3 | Spanish +; + +############################################### +# Bugfixes +############################################### + +multipleBatchesWithSort +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort +required_capability: make_number_of_channels_consistent_with_layout + +from * +| rename city.country.continent.planet.name as message +| lookup join message_types_lookup on message +| sort language_code, birth_date +| keep language_code +| limit 1 +; + +language_code:integer +1 +; + +multipleBatchesWithMvExpand +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort +required_capability: make_number_of_channels_consistent_with_layout + +from * +| rename city.country.continent.planet.name as message +| lookup join message_types_lookup on message +| keep birth_date, language_code +| mv_expand birth_date +| sort birth_date, language_code +| limit 1 +; + +birth_date:datetime |language_code:integer +1952-02-27T00:00:00.000Z |null +; + +multipleBatchesWithAggregate1 +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort +required_capability: make_number_of_channels_consistent_with_layout + +from * +| rename city.country.continent.planet.name as message +| lookup join message_types_lookup on message +| keep birth_date, language_code +| stats x=max(birth_date), y=min(language_code) +; + +x:datetime |y:integer +1965-01-03T00:00:00.000Z |1 +; + +multipleBatchesWithAggregate2 +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort +required_capability: make_number_of_channels_consistent_with_layout + +from * +| rename city.country.continent.planet.name as message +| lookup join message_types_lookup on message +| keep birth_date, language_code +| stats m=min(birth_date) by language_code +| sort language_code +| limit 1 +; + +m:datetime |language_code:integer +null |1 +; + +multipleBatchesWithAggregate3 +required_capability: join_lookup_v12 +required_capability: remove_redundant_sort +required_capability: make_number_of_channels_consistent_with_layout + +from * +| rename city.country.continent.planet.name as message +| lookup join message_types_lookup on message +| keep birth_date, language_code +| stats m=min(language_code) by birth_date +| sort birth_date +| limit 1 +; + +m:integer |birth_date:datetime +null |1952-02-27T00:00:00.000Z +; + +// Regression test for https://github.com/elastic/elasticsearch/issues/126030 +// We had wrong layouts from ReplaceMissingFieldsWithNull + +enrichLookupStatsBug +required_capability: join_lookup_v12 +required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout + +from * +| enrich languages_policy on cluster +| rename languages.byte as language_code +| lookup join languages_lookup on language_code +| stats salary_change.long = max(ratings), foo = max(num) +; + +salary_change.long:double|foo:long +5.0 |1698069301543123456 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json index db15133f036bb..5fa25e01ef0e4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json @@ -76,6 +76,9 @@ "st_logs": { "type": "semantic_text", "inference_id": "test_sparse_inference" + }, + "language_name": { + "type": "keyword" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec index e6a63d1078d97..bf27ee0d33167 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec @@ -721,3 +721,39 @@ book_no:keyword 7140 2714 ; + +testMatchWithSemanticTextAndLookupJoin +required_capability: match_operator_colon +required_capability: semantic_text_field_caps +required_capability: join_lookup_v12 + +from semantic_text METADATA _id +| where semantic_text_field:"something" +| LOOKUP JOIN languages_lookup ON language_name +| KEEP _id, language_name, language_code +; + +ignoreOrder: true + +_id:keyword | language_name:keyword | language_code:integer +1 | English | 1 +2 | French | 2 +3 | Spanish | 3 +; + +testMatchWithSemanticTextKqlAndLookupJoin +required_capability: match_operator_colon +required_capability: semantic_text_type +required_capability: kql_function +required_capability: semantic_text_field_caps +required_capability: join_lookup_v12 + +from semantic_text +| where kql("host:host1") AND semantic_text_field:"something" +| LOOKUP JOIN languages_lookup ON language_name +| KEEP host, semantic_text_field, language_name, language_code +; + +host:keyword | semantic_text_field:text | language_name:keyword | language_code:integer +"host1" | live long and prosper | English | 1 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec index ec00b613e2e0b..755484fc35540 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec @@ -1603,3 +1603,95 @@ emp_no: integer | x:date | y:date 10001 | 2024-11-03 | 2024-11-06 10002 | 2024-11-03 | 2024-11-06 ; + +greatestWithMixedNumericValues +required_capability: mixed_numeric_types_in_case_greatest_least +ROW g1=GREATEST(10.0, 5.0, 1, -100.1, 0, 1234, -10000), g2=GREATEST(10.0, 5, 1, -100.1, null); + +g1:double |g2:double +1234 |null +; + +leastWithMixedNumericValues +required_capability: mixed_numeric_types_in_case_greatest_least +ROW l1=LEAST(10.0, 5.0, 1, -100.1, 0, 1234, -10000), l2=LEAST(10.0, 5, 1, -100.1, null); + +l1:double |l2:double +-10000 |null +; + +greatestWithMixedNumericFields +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL g1 = GREATEST(height, salary, languages), g2 = GREATEST(height, salary, languages, null) +| KEEP emp_no, g1, g2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | g1:double | g2:double +10001 | 57305.0 | null +10002 | 56371.0 | null +10003 | 61805.0 | null +; + +leastWithMixedNumericFields +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL l1 = LEAST(height, salary, languages), l2 = LEAST(height, salary, languages, null) +| KEEP emp_no, l1, l2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | l1:double | l2:double +10001 | 2.0 | null +10002 | 2.08 | null +10003 | 1.83 | null +; + +greatestWithMixedNumericValuesWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +ROW g1=GREATEST([10.0, 4], 1), g2=GREATEST([10.0, 4], 1, null); + +g1:double |g2:double +10 |null +; + +leastWithMixedNumericValuesWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +ROW l1=LEAST([10.0, 4], 1), l2=LEAST([10.0, 4], 1, null); + +l1:double |l2:double +1 |null +; + +greatestWithMixedNumericFieldsWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL g1 = GREATEST(salary_change, salary, languages), g2 = GREATEST(salary_change, salary, languages, null) +| KEEP emp_no, g1, g2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | g1:double | g2:double +10001 | 57305.0 | null +10002 | 56371.0 | null +10003 | 61805.0 | null +; + +leastWithMixedNumericFieldsWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL l1 = LEAST(salary_change, salary, languages), l2 = LEAST(salary_change, salary, languages, null) +| KEEP emp_no, l1, l2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | l1:double | l2:double +10001 | 1.19 | null +10002 | -7.23 | null +10003 | 4.0 | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index 2a7c092798404..1b4c1f0bc2b6c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -404,3 +404,17 @@ from employees | where emp_no == 10003 | mv_expand first_name | keep first_name first_name:keyword Parto ; + + +sortBeforeAndAfterMvExpand +from employees +| sort first_name +| mv_expand job_positions +| sort emp_no, job_positions +| keep emp_no, job_positions +| limit 2; + +emp_no:integer | job_positions:keyword +10001 | Accountant +10001 | Senior Python Developer +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 5e23917222345..d10a6178829e6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -525,6 +525,8 @@ POINT (42.97109629958868 14.7552534006536) | 1 stExtentSingleGeoPoint required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") | STATS extent = ST_EXTENT_AGG(point) ; @@ -535,6 +537,8 @@ BBOX(42.97109629958868, 42.97109629958868, 14.7552534006536, 14.7552534006536) stExtentMultipleGeoPoints required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + // tag::st_extent_agg-airports[] FROM airports | WHERE country == "India" @@ -548,35 +552,257 @@ BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) // end::st_extent_agg-airports-result[] ; -stExtentMultipleGeoPointsNoDocValues +stExtentMultipleGeoPointsCount required_capability: st_extent_agg -FROM airports_no_doc_values | WHERE country == "India" | STATS extent = ST_EXTENT_AGG(location) +required_capability: st_extent_agg_docvalues + +FROM airports +| WHERE country == "India" +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() ; -extent:geo_shape -BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) +extent:geo_shape | count:long +BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | 50 +; + +stExtentMultipleGeoPointsCountNoDocValues +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airports_no_doc_values +| WHERE country == "India" +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() +; + +extent:geo_shape | count:long +BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | 50 ; stExtentMultipleGeoPointGrouping required_capability: st_extent_agg -FROM airports | STATS extent = ST_EXTENT_AGG(location) BY country | SORT country | LIMIT 3 +required_capability: st_extent_agg_docvalues + +FROM airports +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() BY country +| SORT count DESC, country ASC +| LIMIT 5 ; -extent:geo_shape | country:keyword -BBOX (69.2100736219436, 69.2100736219436, 34.56339786294848, 34.56339786294848) | Afghanistan -BBOX (19.715032372623682, 19.715032372623682, 41.4208514476195, 41.4208514476195) | Albania -BBOX (-0.6067969836294651, 6.621946580708027, 36.69972063973546, 35.62027471605688) | Algeria +extent:geo_shape | count:long | country:keyword +BBOX (-159.34908430092037, -71.01640669628978, 64.81809809803963, 19.71479767933488) | 129 | United States +BBOX (70.77995480038226, 91.5882289968431, 33.9830909203738, 8.47650992218405) | 50 | India +BBOX (-117.19751106575131, -86.87441730871797, 32.833958650007844, 14.791128113865852) | 45 | Mexico +BBOX (76.01301474496722, 130.45620465651155, 46.84301500674337, 18.309095981530845) | 41 | China +BBOX (-135.07621010765433, -52.743333745747805, 63.751152316108346, 43.163360520266) | 37 | Canada ; stExtentGeoShapes required_capability: st_extent_agg -FROM airport_city_boundaries | WHERE region == "City of New York" | STATS extent = ST_EXTENT_AGG(city_boundary) +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| STATS extent = ST_EXTENT_AGG(city_boundary) ; extent:geo_shape BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) ; +stExtentGeoPoints +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| STATS extent = ST_EXTENT_AGG(city_location) +; + +extent:geo_shape +BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +; + +stExtentGeoShapesAndPoints +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location) +; + +extent_shapes:geo_shape | extent_points:geo_shape +BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +; + +stExtentGeoShapesGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_boundary) BY prefix +| KEEP prefix, extent +| SORT prefix ASC +; + +prefix:keyword | extent:geo_shape +E | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +J | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) +; + +stExtentGeoPointsGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent +| SORT prefix ASC +; + +prefix:keyword | extent:geo_shape +E | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +J | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +L | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +; + +stExtentGeoShapesAndPointsGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| WHERE region == "City of New York" +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent_shapes, extent_points +| SORT prefix ASC +; + +prefix:keyword | extent_shapes:geo_shape | extent_points:geo_shape +E | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +J | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +L | BBOX (-74.25880000926554, -73.70020005851984, 40.91759996954352, 40.47659996431321) | BBOX (-73.92490002326667, -73.92490002326667, 40.69429999217391, 40.69429999217391) +; + +stExtentManyGeoShapesGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_boundary) BY prefix +| KEEP prefix, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent:geo_shape +A | BBOX (-171.91890003159642, 175.90319998562336, 64.61419996339828, -37.36450002528727) +B | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) +C | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) +; + +stExtentManyGeoPointsGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent:geo_shape +A | BBOX (-171.75000007264316, 174.73999994806945, 64.54999999143183, -36.84060002211481) +B | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +C | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +; + +stExtentManyGeoShapesAndPointsGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airport_city_boundaries +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location) BY prefix +| KEEP prefix, extent_shapes, extent_points +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent_shapes:geo_shape | extent_points:geo_shape +A | BBOX (-171.91890003159642, 175.90319998562336, 64.61419996339828, -37.36450002528727) | BBOX (-171.75000007264316, 174.73999994806945, 64.54999999143183, -36.84060002211481) +B | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +C | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +; + +stExtentManyGeoShapesGroupedEnrich +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues +required_capability: enrich_load + +FROM airports +| ENRICH city_boundaries ON city_location WITH airport, region, city_boundary +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_boundary), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT count DESC, prefix ASC +| LIMIT 3 +; + +prefix:keyword | count:long | extent:geo_shape +S | 77 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) +C | 75 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) +B | 69 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) +; + +stExtentManyGeoPointsGroupedEnrich +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues +required_capability: enrich_load + +FROM airports +| ENRICH city_boundaries ON city_location WITH airport, region, city_boundary +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT count DESC, prefix ASC +| LIMIT 3 +; + +prefix:keyword | count:long | extent:geo_shape +S | 77 | BBOX (-135.3152000438422, 178.54539999738336, 69.21669997740537, -33.8678000215441) +C | 75 | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +B | 69 | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +; + +stExtentManyGeoShapesAndPointsGroupedEnrich +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues +required_capability: enrich_load + +FROM airports +| ENRICH city_boundaries ON city_location WITH airport, region, city_boundary +| EVAL prefix = SUBSTRING(abbrev, 1, 1) +| STATS extent_shapes = ST_EXTENT_AGG(city_boundary), extent_points = ST_EXTENT_AGG(city_location), count = COUNT() BY prefix +| KEEP prefix, count, extent_shapes, extent_points +| SORT count DESC, prefix ASC +| LIMIT 3 +; + +prefix:keyword | count:long | extent_shapes:geo_shape | extent_points:geo_shape +S | 77 | BBOX (-136.45440001040697, 178.8686999771744, 61.38089996762574, -33.92440003808588) | BBOX (-135.3152000438422, 178.54539999738336, 69.21669997740537, -33.8678000215441) +C | 75 | BBOX (-107.51820000819862, 172.6055999379605, 55.732699991203845, -43.90400002710521) | BBOX (-107.39390007220209, 172.38329996354878, 55.676099974662066, -43.58330002985895) +B | 69 | BBOX (-116.51340007781982, 153.2021999359131, 60.631899973377585, -41.20620000176132) | BBOX (-116.23080002143979, 153.02809992805123, 60.46669999603182, -41.1500000115484) +; + ############################################### # Tests for ST_INTERSECTS on GEO_POINT type @@ -1778,6 +2004,18 @@ extent:cartesian_shape BBOX (4783520.5, 1.6168486E7, 8704352.0, -584415.9375) ; +stExtentMultipleCartesianPointsCount +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM airports_web +| STATS extent = ST_EXTENT_AGG(location), count = COUNT() +; + +extent:cartesian_shape | count:long +BBOX (-1.949601E7, 1.9947946E7, 1.4502138E7, -7128878.5) | 849 +; + stExtentMultipleCartesianPointGrouping required_capability: st_extent_agg FROM airports_web | STATS extent = ST_EXTENT_AGG(location) BY scalerank | SORT scalerank DESC | LIMIT 3 @@ -1839,6 +2077,42 @@ count:long | key:keyword | extent:cartesian_shape 4 | Fou | BBOX (0.0, 3.0, 3.0, 0.0) ; +stExtentManyCartesianShapesGrouped +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM countries_bbox_web +| EVAL prefix = SUBSTRING(id, 1, 1) +| STATS extent = ST_EXTENT_AGG(shape) BY prefix +| KEEP prefix, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | extent:cartesian_shape +A | BBOX (-2.0037508E7, 2.0037508E7, 6278042.5, -4.748140544E9) +B | BBOX (-9931524.0, 1.2841846E7, 7591831.0, -3994093.25) +C | BBOX (-1.8462154E7, 1.5002357E7, 1.7926778E7, -7538976.5) +; + +stExtentManyCartesianShapesGroupedCount +required_capability: st_extent_agg +required_capability: st_extent_agg_docvalues + +FROM countries_bbox_web +| EVAL prefix = SUBSTRING(id, 1, 1) +| STATS extent = ST_EXTENT_AGG(shape), count = COUNT() BY prefix +| KEEP prefix, count, extent +| SORT prefix +| LIMIT 3 +; + +prefix:keyword | count:long | extent:cartesian_shape +A | 17 | BBOX (-2.0037508E7, 2.0037508E7, 6278042.5, -4.748140544E9) +B | 18 | BBOX (-9931524.0, 1.2841846E7, 7591831.0, -3994093.25) +C | 19 | BBOX (-1.8462154E7, 1.5002357E7, 1.7926778E7, -7538976.5) +; + ############################################### # Tests for ST_INTERSECTS on CARTESIAN_POINT type diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index f91ed3e9b08b0..7e8ab52854f56 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -964,6 +964,47 @@ false | null false | null ; +startsWithLucenePushdown + +from hosts +| where starts_with(host, "bet") and starts_with(host_group, "Kuber") +| keep host, host_group +| sort host, host_group; + +host:keyword | host_group:text +beta | Kubernetes cluster +beta | Kubernetes cluster +beta | Kubernetes cluster +; + +startsWithLuceneDisabledPushdown + +from hosts +| where host == "unknown host" or (starts_with(host, "bet") and starts_with(host_group, "Kuber")) +| keep host, host_group +| sort host, host_group; + +host:keyword | host_group:text +beta | Kubernetes cluster +beta | Kubernetes cluster +beta | Kubernetes cluster +; + +startsWithLucenePushdownIgnoreMultivalues +required_capability: starts_with_ends_with_lucene_pushdown + +from hosts +| where starts_with(description, "epsilon") +| keep description +| sort description; + +warningRegex: evaluation of \[starts_with\(description, \\\"epsilon\\\"\)\] failed, treating result as null. Only first 20 failures recorded. +warningRegex: java.lang.IllegalArgumentException: single-value function encountered multi-value + +description:text +epsilon gw instance +; + substringOfText required_capability: mv_warn @@ -1192,6 +1233,144 @@ Bernatsky |false ; +endsWithLucenePushdown + +from hosts +| where ends_with(host, "ta") and ends_with(host_group, "cluster") +| keep host, host_group +| sort host, host_group; + +host:keyword | host_group:text +beta | Kubernetes cluster +beta | Kubernetes cluster +beta | Kubernetes cluster +; + +endsWithLuceneDisabledPushdown + +from hosts +| where host == "unknown host" or (ends_with(host, "ta") and ends_with(host_group, "cluster")) +| keep host, host_group +| sort host, host_group; + +host:keyword | host_group:text +beta | Kubernetes cluster +beta | Kubernetes cluster +beta | Kubernetes cluster +; + +endsWithLucenePushdownIgnoreMultivalues +required_capability: starts_with_ends_with_lucene_pushdown + +from hosts +| where ends_with(description, "host") +| keep description +| sort description; + +warningRegex: evaluation of \[ends_with\(description, \\\"host\\\"\)\] failed, treating result as null. Only first 20 failures recorded. +warningRegex: java.lang.IllegalArgumentException: single-value function encountered multi-value + +description:text +; + + +lucenePushdownMultipleWhere + +from hosts +| where starts_with(host, "bet") +| keep host, host_group +| sort host, host_group +| where ends_with(host_group, "cluster"); + +host:keyword | host_group:text +beta | Kubernetes cluster +beta | Kubernetes cluster +beta | Kubernetes cluster +; + +lucenePushdownMultipleIndices +required_capability: casting_operator + +from airports* +| where starts_with(name::keyword, "Sahn") and ends_with(abbrev, "UH") +| keep abbrev, name +| sort abbrev, name; + +abbrev:keyword | name:text +LUH | Sahnewal +LUH | Sahnewal +LUH | Sahnewal +LUH | Sahnewal +LUH | Sahnewal +LUH | Sahnewal +; + +lucenePushdownOr +required_capability: casting_operator + +from airports +| where starts_with(name::keyword, "Sahn") or ends_with(abbrev, "UH") +| keep abbrev, name +| sort abbrev, name; + +abbrev:keyword | name:text +AUH | Abu Dhabi Int'l +LUH | Sahnewal +RUH | King Khalid Int'l +; + +lucenePushdownMultipleOr +required_capability: casting_operator + +from airports +| where starts_with(name::keyword, "Sahn") or ends_with(abbrev, "UH") or starts_with(abbrev, "OOL") +| keep abbrev, name +| sort abbrev, name; + +abbrev:keyword | name:text +AUH | Abu Dhabi Int'l +LUH | Sahnewal +OOL | Gold Coast +RUH | King Khalid Int'l +; + +lucenePushdownMultipleAnd +required_capability: casting_operator + +from airports +| where starts_with(name::keyword, "Sahn") and ends_with(abbrev, "UH") +| where ends_with(name::keyword, "al") +| keep abbrev, name +| sort abbrev, name; + +abbrev:keyword | name:text +LUH | Sahnewal +; + +lucenePushdownMixAndOr +required_capability: casting_operator + +from airports +| where starts_with(name::keyword, "Sahn") and (starts_with(name::keyword, "Abc") or ends_with(abbrev, "UH")) +| keep abbrev, name, scalerank +| sort abbrev, name; + +abbrev:keyword | name:text | scalerank:integer +LUH | Sahnewal | 9 +; + +lucenePushdownMixOrAnd +required_capability: casting_operator + +from airports +| where starts_with(name::keyword, "Sahn") or (starts_with(abbrev, "G") and ends_with(name::keyword, "Falls Int'l")) +| keep abbrev, name, scalerank +| sort abbrev; + +abbrev:keyword | name:text | scalerank:integer +GTF | Great Falls Int'l | 8 +LUH | Sahnewal | 9 +; toLowerRow#[skip:-8.12.99] // tag::to_lower[] diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java index 99a81c60a9ad2..049a4cbec3b20 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncEnrichStopIT.java @@ -8,9 +8,12 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.compute.operator.DriverTaskRunner; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.async.AsyncStopRequest; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -31,7 +34,9 @@ import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.deleteAsyncId; import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.startAsyncQuery; import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.waitForCluster; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; // This tests if enrich after stop works correctly public class CrossClusterAsyncEnrichStopIT extends AbstractEnrichBasedCrossClusterTestCase { @@ -87,10 +92,23 @@ public void testEnrichAfterStop() throws Exception { // wait until c1 is done waitForCluster(client(), "c1", asyncExecutionId); waitForCluster(client(), LOCAL_CLUSTER, asyncExecutionId); + // wait until remote reduce task starts on c2 + assertBusy(() -> { + List tasks = getDriverTasks(client(REMOTE_CLUSTER_2)); + List reduceTasks = tasks.stream().filter(t -> t.description().contains("_LuceneSourceOperator") == false).toList(); + assertThat(reduceTasks, not(empty())); + }); // Run the stop request var stopRequest = new AsyncStopRequest(asyncExecutionId); var stopAction = client().execute(EsqlAsyncStopAction.INSTANCE, stopRequest); + // wait until remote reduce tasks are gone + assertBusy(() -> { + List tasks = getDriverTasks(client(REMOTE_CLUSTER_2)); + List reduceTasks = tasks.stream().filter(t -> t.description().contains("_LuceneSourceOperator") == false).toList(); + assertThat(reduceTasks, empty()); + }); + // Allow the processing to proceed SimplePauseFieldPlugin.allowEmitting.countDown(); @@ -153,4 +171,9 @@ record Event(long timestamp, String user, String host) {} } client.admin().indices().prepareRefresh("events").get(); } + + static List getDriverTasks(Client client) { + return client.admin().cluster().prepareListTasks().setActions(DriverTaskRunner.ACTION_NAME).setDetailed(true).get().getTasks(); + } + } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index de9eb166688f9..c21d92039d854 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -587,6 +587,28 @@ public void testFilterWithNullAndEval() { } } + public void testSortWithNull() { + try (EsqlQueryResponse results = run("row a = null | sort a")) { + logger.info(results); + assertEquals(1, getValuesList(results).size()); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("a", "null")); + assertThat(results.columns().stream().map(ColumnInfo::name).toList(), contains("a")); + assertThat(results.columns().stream().map(ColumnInfoImpl::type).toList(), contains(DataType.NULL)); + assertNull(getValuesList(results).get(0).get(countIndex)); + } + } + + public void testStatsByNull() { + try (EsqlQueryResponse results = run("row a = null | stats by a")) { + logger.info(results); + assertEquals(1, getValuesList(results).size()); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("a", "null")); + assertThat(results.columns().stream().map(ColumnInfo::name).toList(), contains("a")); + assertThat(results.columns().stream().map(ColumnInfoImpl::type).toList(), contains(DataType.NULL)); + assertNull(getValuesList(results).get(0).get(countIndex)); + } + } + public void testStringLength() { try (EsqlQueryResponse results = run("from test | eval l = length(color)")) { logger.info(results); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index 2d0a15436bf82..85c03ce7860d3 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -462,7 +462,9 @@ protected void doRun() throws Exception { } Exception failure = expectThrows(Exception.class, () -> future.actionGet().close()); EsqlTestUtils.assertEsqlFailure(failure); - assertThat(failure.getMessage(), containsString("failed to fetch pages")); + Throwable cause = ExceptionsHelper.unwrap(failure, IOException.class); + assertNotNull(cause); + assertThat(cause.getMessage(), containsString("failed to fetch pages")); // If we proceed without waiting for pages, we might cancel the main request before starting the data-node request. // As a result, the exchange sinks on data-nodes won't be removed until the inactive_timeout elapses, which is // longer than the assertBusy timeout. diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java index 1e34421097aac..1118121b0becb 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlNodeFailureIT.java @@ -8,7 +8,9 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.compute.operator.exchange.ExchangeService; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.FailingFieldPlugin; @@ -27,9 +29,23 @@ */ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class EsqlNodeFailureIT extends AbstractEsqlIntegTestCase { + @Override protected Collection> nodePlugins() { - return CollectionUtils.appendToCopy(super.nodePlugins(), FailingFieldPlugin.class); + var plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(FailingFieldPlugin.class); + plugins.add(InternalExchangePlugin.class); + return plugins; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings settings = Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMillis(between(3000, 4000))) + .build(); + logger.info("settings {}", settings); + return settings; } /** @@ -49,7 +65,7 @@ public void testFailureLoadingFields() throws IOException { mapping.endObject(); client().admin().indices().prepareCreate("fail").setSettings(indexSettings(1, 0)).setMapping(mapping.endObject()).get(); - int docCount = 100; + int docCount = 50; List docs = new ArrayList<>(docCount); for (int d = 0; d < docCount; d++) { docs.add(client().prepareIndex("ok").setSource("foo", d)); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java index 7036216ebbbcf..c8dc134e0e706 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java @@ -142,11 +142,9 @@ public static Iterable parameters() { | EVAL y = to_str(host) | LOOKUP JOIN lookup_idx ON host """, - Build.current().isSnapshot() - ? Map.ofEntries(Map.entry("FROM", 1), Map.entry("EVAL", 1), Map.entry("LOOKUP JOIN", 1)) - : Collections.emptyMap(), - Build.current().isSnapshot() ? Map.ofEntries(Map.entry("TO_STRING", 1)) : Collections.emptyMap(), - Build.current().isSnapshot() + Map.ofEntries(Map.entry("FROM", 1), Map.entry("EVAL", 1), Map.entry("LOOKUP JOIN", 1)), + Map.ofEntries(Map.entry("TO_STRING", 1)), + true ) }, new Object[] { new Test( diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java index bd7246518c958..9faad45712527 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -18,6 +20,8 @@ import java.util.List; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.containsString; @@ -122,6 +126,119 @@ public void testWhereMatchWithScoring() { } } + /** + * Test for https://github.com/elastic/elasticsearch/issues/123967 + */ + public void testWhereMatchWithScoring_AndRequestFilter() { + var query = """ + FROM test METADATA _score + | WHERE content:"fox" + | SORT _score DESC + | KEEP content, _score + """; + + QueryBuilder filter = boolQuery().must(matchQuery("content", "brown")); + + try (var resp = run(query, randomPragmas(), filter)) { + assertColumnNames(resp.columns(), List.of("content", "_score")); + assertColumnTypes(resp.columns(), List.of("text", "double")); + assertValues( + resp.values(), + List.of( + List.of("This is a brown fox", 1.4274532794952393), + List.of("The quick brown fox jumps over the lazy dog", 1.1248724460601807) + ) + ); + } + } + + public void testWhereMatchWithScoring_AndNoScoreRequestFilter() { + var query = """ + FROM test METADATA _score + | WHERE content:"fox" + | SORT _score DESC + | KEEP content, _score + """; + + QueryBuilder filter = boolQuery().filter(matchQuery("content", "brown")); + + try (var resp = run(query, randomPragmas(), filter)) { + assertColumnNames(resp.columns(), List.of("content", "_score")); + assertColumnTypes(resp.columns(), List.of("text", "double")); + assertValues( + resp.values(), + List.of( + List.of("This is a brown fox", 1.156558871269226), + List.of("The quick brown fox jumps over the lazy dog", 0.9114001989364624) + ) + ); + } + } + + public void testWhereMatchWithScoring_And_MatchAllRequestFilter() { + var query = """ + FROM test METADATA _score + | WHERE content:"fox" + | SORT _score DESC + | KEEP content, _score + """; + + QueryBuilder filter = QueryBuilders.matchAllQuery(); + + try (var resp = run(query, randomPragmas(), filter)) { + assertColumnNames(resp.columns(), List.of("content", "_score")); + assertColumnTypes(resp.columns(), List.of("text", "double")); + assertValues( + resp.values(), + List.of( + List.of("This is a brown fox", 2.1565589904785156), + List.of("The quick brown fox jumps over the lazy dog", 1.9114001989364624) + ) + ); + } + } + + public void testScoringOutsideQuery() { + var query = """ + FROM test METADATA _score + | SORT _score DESC + | KEEP content, _score + """; + + QueryBuilder filter = boolQuery().must(matchQuery("content", "fox")); + + try (var resp = run(query, randomPragmas(), filter)) { + assertColumnNames(resp.columns(), List.of("content", "_score")); + assertColumnTypes(resp.columns(), List.of("text", "double")); + assertValues( + resp.values(), + List.of( + List.of("This is a brown fox", 1.156558871269226), + List.of("The quick brown fox jumps over the lazy dog", 0.9114001989364624) + ) + ); + } + } + + public void testScoring_Zero_OutsideQuery() { + var query = """ + FROM test METADATA _score + | SORT content DESC + | KEEP content, _score + """; + + QueryBuilder filter = boolQuery().filter(matchQuery("content", "fox")); + + try (var resp = run(query, randomPragmas(), filter)) { + assertColumnNames(resp.columns(), List.of("content", "_score")); + assertColumnTypes(resp.columns(), List.of("text", "double")); + assertValues( + resp.values(), + List.of(List.of("This is a brown fox", 0.0), List.of("The quick brown fox jumps over the lazy dog", 0.0)) + ); + } + } + public void testWhereMatchWithScoringDifferentSort() { assumeTrue("'METADATA _score' is disabled", EsqlCapabilities.Cap.METADATA_SCORE.isEnabled()); var query = """ diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationIT.java new file mode 100644 index 0000000000000..ab607890eb2ef --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.esql.action.EsqlPluginWithEnterpriseOrTrialLicense; +import org.elasticsearch.xpack.spatial.SpatialPlugin; + +import java.util.Collection; +import java.util.List; + +public class SpatialExtentAggregationIT extends SpatialExtentAggregationTestCase { + @Override + protected Collection> nodePlugins() { + return List.of(SpatialPlugin.class, EsqlPluginWithEnterpriseOrTrialLicense.class); + } + + @Override + public void testStExtentAggregationWithShapes() { + assertStExtentFromIndex("index_geo_shape"); + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationNoLicenseIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationNoLicenseIT.java new file mode 100644 index 0000000000000..6d3a28b9060ee --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationNoLicenseIT.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.spatial.SpatialPlugin; + +import java.util.Collection; +import java.util.List; + +public class SpatialExtentAggregationNoLicenseIT extends SpatialExtentAggregationTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(TestSpatialPlugin.class, TestEsqlPlugin.class); + } + + @Override + public void testStExtentAggregationWithShapes() { + assertStExtentFailsWith("index_geo_shape"); + } + + private static XPackLicenseState getLicenseState() { + License.OperationMode operationMode; + boolean active; + if (randomBoolean()) { + operationMode = randomFrom( + License.OperationMode.GOLD, + License.OperationMode.BASIC, + License.OperationMode.MISSING, + License.OperationMode.STANDARD + ); + active = true; + } else { + operationMode = randomFrom(License.OperationMode.PLATINUM, License.OperationMode.ENTERPRISE, License.OperationMode.TRIAL); + active = false; // expired + } + + return new XPackLicenseState( + () -> System.currentTimeMillis(), + new XPackLicenseStatus(operationMode, active, "Test license expired") + ); + } + + public static class TestEsqlPlugin extends EsqlPlugin { + protected XPackLicenseState getLicenseState() { + return SpatialExtentAggregationNoLicenseIT.getLicenseState(); + } + } + + public static class TestSpatialPlugin extends SpatialPlugin { + protected XPackLicenseState getLicenseState() { + return SpatialExtentAggregationNoLicenseIT.getLicenseState(); + } + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationTestCase.java new file mode 100644 index 0000000000000..2299d397fbbdd --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialExtentAggregationTestCase.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.junit.Before; + +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public abstract class SpatialExtentAggregationTestCase extends AbstractEsqlIntegTestCase { + + @Before + public void setupIndex() throws Exception { + assumeTrue("requires ST_EXTENT_AGG capability", EsqlCapabilities.Cap.ST_EXTENT_AGG.isEnabled()); + createAndPopulateIndexes(-10, 10, -10, 10); + } + + /** + * This test should pass only with an enterprise license + */ + public abstract void testStExtentAggregationWithShapes(); + + /** + * This test should pass with and without enterprise licenses + */ + public void testStExtentAggregationWithPoints() throws Exception { + assertStExtentFromIndex("index_geo_point"); + } + + protected void assertStExtentFromIndex(String index) { + var query = String.format(Locale.ROOT, """ + FROM %s + | STATS extent = ST_EXTENT_AGG(location) + | EVAL minX = ROUND(ST_XMIN(extent)) + | EVAL maxX = ROUND(ST_XMAX(extent)) + | EVAL minY = ROUND(ST_YMIN(extent)) + | EVAL maxY = ROUND(ST_YMAX(extent)) + """, index); + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("extent", "minX", "maxX", "minY", "maxY")); + assertColumnTypes(resp.columns(), List.of("geo_shape", "double", "double", "double", "double")); + List> values = getValuesList(resp.values()); + assertThat(values.size(), equalTo(1)); + List row = values.get(0); + List expectedValues = List.of(-10.0, 10.0, -10.0, 10.0); + assertThat(row.subList(1, row.size()), equalTo(expectedValues)); + } + } + + protected void assertStExtentFailsWith(String index) { + var query = String.format(Locale.ROOT, """ + FROM %s + | STATS extent = ST_EXTENT_AGG(location) + | EVAL minX = ROUND(ST_XMIN(extent)) + | EVAL maxX = ROUND(ST_XMAX(extent)) + | EVAL minY = ROUND(ST_YMIN(extent)) + | EVAL maxY = ROUND(ST_YMAX(extent)) + """, index); + ElasticsearchException e = expectThrows(VerificationException.class, () -> run(query)); + assertThat(e.getMessage(), containsString("current license is non-compliant for [ST_EXTENT_AGG(location)]")); + } + + private void createAndPopulateIndexes(double minX, double maxX, double minY, double maxY) throws Exception { + int numX = 21; + int numY = 21; + initIndex("index_", "geo_point"); + initIndex("index_", "geo_shape"); + BulkRequestBuilder points = client().prepareBulk(); + BulkRequestBuilder shapes = client().prepareBulk(); + for (int xi = 0; xi < numX; xi++) { + for (int yi = 0; yi < numY; yi++) { + double x = minX + xi * (maxX - minX) / (numX - 1); + double y = minY + yi * (maxY - minY) / (numY - 1); + String point = "POINT(" + x + " " + y + ")"; + points.add(new IndexRequest("index_geo_point").id(x + ":" + y).source("location", point)); + if (xi > 0 && yi > 0) { + double px = minX + (xi - 1) * (maxX - minX) / numX; + double py = minY + (yi - 1) * (maxY - minY) / numY; + String shape = "BBOX(" + px + ", " + x + ", " + y + ", " + py + ")"; + shapes.add(new IndexRequest("index_geo_shape").id(x + ":" + y).source("location", shape)); + } + } + } + points.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + shapes.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + ensureYellow("index_geo_point"); + ensureYellow("index_geo_shape"); + } + + protected void initIndex(String prefix, String fieldType) { + assertAcked(prepareCreate(prefix + fieldType).setMapping(String.format(Locale.ROOT, """ + { + "properties" : { + "location": { "type" : "%s" } + } + } + """, fieldType))); + } +} diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 5b731b5dac9d2..7134fe9a1eebe 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -73,6 +73,7 @@ SHOW : 'show' -> pushMode(SHOW_MODE); SORT : 'sort' -> pushMode(EXPRESSION_MODE); STATS : 'stats' -> pushMode(EXPRESSION_MODE); WHERE : 'where' -> pushMode(EXPRESSION_MODE); +JOIN_LOOKUP : 'lookup' -> pushMode(JOIN_MODE); // // in development // @@ -88,11 +89,9 @@ DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_ DEV_LOOKUP : {this.isDevVersion()}? 'lookup_🐔' -> pushMode(LOOKUP_MODE); DEV_METRICS : {this.isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE); // list of all JOIN commands -DEV_JOIN : {this.isDevVersion()}? 'join' -> pushMode(JOIN_MODE); DEV_JOIN_FULL : {this.isDevVersion()}? 'full' -> pushMode(JOIN_MODE); DEV_JOIN_LEFT : {this.isDevVersion()}? 'left' -> pushMode(JOIN_MODE); DEV_JOIN_RIGHT : {this.isDevVersion()}? 'right' -> pushMode(JOIN_MODE); -DEV_JOIN_LOOKUP : {this.isDevVersion()}? 'lookup' -> pushMode(JOIN_MODE); // @@ -315,8 +314,8 @@ mode PROJECT_MODE; PROJECT_PIPE : PIPE -> type(PIPE), popMode; PROJECT_DOT: DOT -> type(DOT); PROJECT_COMMA : COMMA -> type(COMMA); -PROJECT_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); -PROJECT_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +PROJECT_PARAM : PARAM -> type(PARAM); +PROJECT_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); fragment UNQUOTED_ID_BODY_WITH_PATTERN : (LETTER | DIGIT | UNDERSCORE | ASTERISK) @@ -350,8 +349,8 @@ RENAME_PIPE : PIPE -> type(PIPE), popMode; RENAME_ASSIGN : ASSIGN -> type(ASSIGN); RENAME_COMMA : COMMA -> type(COMMA); RENAME_DOT: DOT -> type(DOT); -RENAME_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); -RENAME_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +RENAME_PARAM : PARAM -> type(PARAM); +RENAME_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); AS : 'as'; @@ -423,8 +422,8 @@ ENRICH_FIELD_QUOTED_IDENTIFIER : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) ; -ENRICH_FIELD_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); -ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +ENRICH_FIELD_PARAM : PARAM -> type(PARAM); +ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); ENRICH_FIELD_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN) @@ -441,8 +440,8 @@ ENRICH_FIELD_WS mode MVEXPAND_MODE; MVEXPAND_PIPE : PIPE -> type(PIPE), popMode; MVEXPAND_DOT: DOT -> type(DOT); -MVEXPAND_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); -MVEXPAND_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +MVEXPAND_PARAM : PARAM -> type(PARAM); +MVEXPAND_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); MVEXPAND_QUOTED_IDENTIFIER : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) @@ -556,7 +555,7 @@ LOOKUP_FIELD_WS // mode JOIN_MODE; JOIN_PIPE : PIPE -> type(PIPE), popMode; -JOIN_JOIN : DEV_JOIN -> type(DEV_JOIN); +JOIN : 'join'; JOIN_AS : AS -> type(AS); JOIN_ON : ON -> type(ON), popMode, pushMode(EXPRESSION_MODE); USING : 'USING' -> popMode, pushMode(EXPRESSION_MODE); diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 218884913960f..02af324872fc0 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -14,110 +14,110 @@ SHOW=13 SORT=14 STATS=15 WHERE=16 -DEV_INLINESTATS=17 -DEV_LOOKUP=18 -DEV_METRICS=19 -DEV_JOIN=20 +JOIN_LOOKUP=17 +DEV_INLINESTATS=18 +DEV_LOOKUP=19 +DEV_METRICS=20 DEV_JOIN_FULL=21 DEV_JOIN_LEFT=22 DEV_JOIN_RIGHT=23 -DEV_JOIN_LOOKUP=24 -UNKNOWN_CMD=25 -LINE_COMMENT=26 -MULTILINE_COMMENT=27 -WS=28 -PIPE=29 -QUOTED_STRING=30 -INTEGER_LITERAL=31 -DECIMAL_LITERAL=32 -BY=33 -AND=34 -ASC=35 -ASSIGN=36 -CAST_OP=37 -COLON=38 -COMMA=39 -DESC=40 -DOT=41 -FALSE=42 -FIRST=43 -IN=44 -IS=45 -LAST=46 -LIKE=47 -LP=48 -NOT=49 -NULL=50 -NULLS=51 -OR=52 -PARAM=53 -RLIKE=54 -RP=55 -TRUE=56 -EQ=57 -CIEQ=58 -NEQ=59 -LT=60 -LTE=61 -GT=62 -GTE=63 -PLUS=64 -MINUS=65 -ASTERISK=66 -SLASH=67 -PERCENT=68 -LEFT_BRACES=69 -RIGHT_BRACES=70 -NAMED_OR_POSITIONAL_PARAM=71 -OPENING_BRACKET=72 -CLOSING_BRACKET=73 -UNQUOTED_IDENTIFIER=74 -QUOTED_IDENTIFIER=75 -EXPR_LINE_COMMENT=76 -EXPR_MULTILINE_COMMENT=77 -EXPR_WS=78 -EXPLAIN_WS=79 -EXPLAIN_LINE_COMMENT=80 -EXPLAIN_MULTILINE_COMMENT=81 -METADATA=82 -UNQUOTED_SOURCE=83 -FROM_LINE_COMMENT=84 -FROM_MULTILINE_COMMENT=85 -FROM_WS=86 -ID_PATTERN=87 -PROJECT_LINE_COMMENT=88 -PROJECT_MULTILINE_COMMENT=89 -PROJECT_WS=90 -AS=91 -RENAME_LINE_COMMENT=92 -RENAME_MULTILINE_COMMENT=93 -RENAME_WS=94 -ON=95 -WITH=96 -ENRICH_POLICY_NAME=97 -ENRICH_LINE_COMMENT=98 -ENRICH_MULTILINE_COMMENT=99 -ENRICH_WS=100 -ENRICH_FIELD_LINE_COMMENT=101 -ENRICH_FIELD_MULTILINE_COMMENT=102 -ENRICH_FIELD_WS=103 -MVEXPAND_LINE_COMMENT=104 -MVEXPAND_MULTILINE_COMMENT=105 -MVEXPAND_WS=106 -INFO=107 -SHOW_LINE_COMMENT=108 -SHOW_MULTILINE_COMMENT=109 -SHOW_WS=110 -SETTING=111 -SETTING_LINE_COMMENT=112 -SETTTING_MULTILINE_COMMENT=113 -SETTING_WS=114 -LOOKUP_LINE_COMMENT=115 -LOOKUP_MULTILINE_COMMENT=116 -LOOKUP_WS=117 -LOOKUP_FIELD_LINE_COMMENT=118 -LOOKUP_FIELD_MULTILINE_COMMENT=119 -LOOKUP_FIELD_WS=120 +UNKNOWN_CMD=24 +LINE_COMMENT=25 +MULTILINE_COMMENT=26 +WS=27 +PIPE=28 +QUOTED_STRING=29 +INTEGER_LITERAL=30 +DECIMAL_LITERAL=31 +BY=32 +AND=33 +ASC=34 +ASSIGN=35 +CAST_OP=36 +COLON=37 +COMMA=38 +DESC=39 +DOT=40 +FALSE=41 +FIRST=42 +IN=43 +IS=44 +LAST=45 +LIKE=46 +LP=47 +NOT=48 +NULL=49 +NULLS=50 +OR=51 +PARAM=52 +RLIKE=53 +RP=54 +TRUE=55 +EQ=56 +CIEQ=57 +NEQ=58 +LT=59 +LTE=60 +GT=61 +GTE=62 +PLUS=63 +MINUS=64 +ASTERISK=65 +SLASH=66 +PERCENT=67 +LEFT_BRACES=68 +RIGHT_BRACES=69 +NAMED_OR_POSITIONAL_PARAM=70 +OPENING_BRACKET=71 +CLOSING_BRACKET=72 +UNQUOTED_IDENTIFIER=73 +QUOTED_IDENTIFIER=74 +EXPR_LINE_COMMENT=75 +EXPR_MULTILINE_COMMENT=76 +EXPR_WS=77 +EXPLAIN_WS=78 +EXPLAIN_LINE_COMMENT=79 +EXPLAIN_MULTILINE_COMMENT=80 +METADATA=81 +UNQUOTED_SOURCE=82 +FROM_LINE_COMMENT=83 +FROM_MULTILINE_COMMENT=84 +FROM_WS=85 +ID_PATTERN=86 +PROJECT_LINE_COMMENT=87 +PROJECT_MULTILINE_COMMENT=88 +PROJECT_WS=89 +AS=90 +RENAME_LINE_COMMENT=91 +RENAME_MULTILINE_COMMENT=92 +RENAME_WS=93 +ON=94 +WITH=95 +ENRICH_POLICY_NAME=96 +ENRICH_LINE_COMMENT=97 +ENRICH_MULTILINE_COMMENT=98 +ENRICH_WS=99 +ENRICH_FIELD_LINE_COMMENT=100 +ENRICH_FIELD_MULTILINE_COMMENT=101 +ENRICH_FIELD_WS=102 +MVEXPAND_LINE_COMMENT=103 +MVEXPAND_MULTILINE_COMMENT=104 +MVEXPAND_WS=105 +INFO=106 +SHOW_LINE_COMMENT=107 +SHOW_MULTILINE_COMMENT=108 +SHOW_WS=109 +SETTING=110 +SETTING_LINE_COMMENT=111 +SETTTING_MULTILINE_COMMENT=112 +SETTING_WS=113 +LOOKUP_LINE_COMMENT=114 +LOOKUP_MULTILINE_COMMENT=115 +LOOKUP_WS=116 +LOOKUP_FIELD_LINE_COMMENT=117 +LOOKUP_FIELD_MULTILINE_COMMENT=118 +LOOKUP_FIELD_WS=119 +JOIN=120 USING=121 JOIN_LINE_COMMENT=122 JOIN_MULTILINE_COMMENT=123 @@ -144,49 +144,51 @@ CLOSING_METRICS_WS=130 'sort'=14 'stats'=15 'where'=16 -'|'=29 -'by'=33 -'and'=34 -'asc'=35 -'='=36 -'::'=37 -':'=38 -','=39 -'desc'=40 -'.'=41 -'false'=42 -'first'=43 -'in'=44 -'is'=45 -'last'=46 -'like'=47 -'('=48 -'not'=49 -'null'=50 -'nulls'=51 -'or'=52 -'?'=53 -'rlike'=54 -')'=55 -'true'=56 -'=='=57 -'=~'=58 -'!='=59 -'<'=60 -'<='=61 -'>'=62 -'>='=63 -'+'=64 -'-'=65 -'*'=66 -'/'=67 -'%'=68 -'{'=69 -'}'=70 -']'=73 -'metadata'=82 -'as'=91 -'on'=95 -'with'=96 -'info'=107 +'lookup'=17 +'|'=28 +'by'=32 +'and'=33 +'asc'=34 +'='=35 +'::'=36 +':'=37 +','=38 +'desc'=39 +'.'=40 +'false'=41 +'first'=42 +'in'=43 +'is'=44 +'last'=45 +'like'=46 +'('=47 +'not'=48 +'null'=49 +'nulls'=50 +'or'=51 +'?'=52 +'rlike'=53 +')'=54 +'true'=55 +'=='=56 +'=~'=57 +'!='=58 +'<'=59 +'<='=60 +'>'=61 +'>='=62 +'+'=63 +'-'=64 +'*'=65 +'/'=66 +'%'=67 +'{'=68 +'}'=69 +']'=72 +'metadata'=81 +'as'=90 +'on'=94 +'with'=95 +'info'=106 +'join'=120 'USING'=121 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index 776cc8e2a9594..5e641de30ccd1 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -51,10 +51,10 @@ processingCommand | grokCommand | enrichCommand | mvExpandCommand + | joinCommand // in development | {this.isDevVersion()}? inlinestatsCommand | {this.isDevVersion()}? lookupCommand - | {this.isDevVersion()}? joinCommand ; whereCommand @@ -202,7 +202,7 @@ identifier identifierPattern : ID_PATTERN - | {this.isDevVersion()}? parameter + | parameter ; constant @@ -225,7 +225,7 @@ parameter identifierOrParameter : identifier - | {this.isDevVersion()}? parameter + | parameter ; limitCommand @@ -333,11 +333,11 @@ inlinestatsCommand ; joinCommand - : type=(DEV_JOIN_LOOKUP | DEV_JOIN_LEFT | DEV_JOIN_RIGHT)? DEV_JOIN joinTarget joinCondition + : type=(JOIN_LOOKUP | DEV_JOIN_LEFT | DEV_JOIN_RIGHT) JOIN joinTarget joinCondition ; joinTarget - : index=indexPattern (AS alias=identifier)? + : index=indexPattern ; joinCondition diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 218884913960f..02af324872fc0 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -14,110 +14,110 @@ SHOW=13 SORT=14 STATS=15 WHERE=16 -DEV_INLINESTATS=17 -DEV_LOOKUP=18 -DEV_METRICS=19 -DEV_JOIN=20 +JOIN_LOOKUP=17 +DEV_INLINESTATS=18 +DEV_LOOKUP=19 +DEV_METRICS=20 DEV_JOIN_FULL=21 DEV_JOIN_LEFT=22 DEV_JOIN_RIGHT=23 -DEV_JOIN_LOOKUP=24 -UNKNOWN_CMD=25 -LINE_COMMENT=26 -MULTILINE_COMMENT=27 -WS=28 -PIPE=29 -QUOTED_STRING=30 -INTEGER_LITERAL=31 -DECIMAL_LITERAL=32 -BY=33 -AND=34 -ASC=35 -ASSIGN=36 -CAST_OP=37 -COLON=38 -COMMA=39 -DESC=40 -DOT=41 -FALSE=42 -FIRST=43 -IN=44 -IS=45 -LAST=46 -LIKE=47 -LP=48 -NOT=49 -NULL=50 -NULLS=51 -OR=52 -PARAM=53 -RLIKE=54 -RP=55 -TRUE=56 -EQ=57 -CIEQ=58 -NEQ=59 -LT=60 -LTE=61 -GT=62 -GTE=63 -PLUS=64 -MINUS=65 -ASTERISK=66 -SLASH=67 -PERCENT=68 -LEFT_BRACES=69 -RIGHT_BRACES=70 -NAMED_OR_POSITIONAL_PARAM=71 -OPENING_BRACKET=72 -CLOSING_BRACKET=73 -UNQUOTED_IDENTIFIER=74 -QUOTED_IDENTIFIER=75 -EXPR_LINE_COMMENT=76 -EXPR_MULTILINE_COMMENT=77 -EXPR_WS=78 -EXPLAIN_WS=79 -EXPLAIN_LINE_COMMENT=80 -EXPLAIN_MULTILINE_COMMENT=81 -METADATA=82 -UNQUOTED_SOURCE=83 -FROM_LINE_COMMENT=84 -FROM_MULTILINE_COMMENT=85 -FROM_WS=86 -ID_PATTERN=87 -PROJECT_LINE_COMMENT=88 -PROJECT_MULTILINE_COMMENT=89 -PROJECT_WS=90 -AS=91 -RENAME_LINE_COMMENT=92 -RENAME_MULTILINE_COMMENT=93 -RENAME_WS=94 -ON=95 -WITH=96 -ENRICH_POLICY_NAME=97 -ENRICH_LINE_COMMENT=98 -ENRICH_MULTILINE_COMMENT=99 -ENRICH_WS=100 -ENRICH_FIELD_LINE_COMMENT=101 -ENRICH_FIELD_MULTILINE_COMMENT=102 -ENRICH_FIELD_WS=103 -MVEXPAND_LINE_COMMENT=104 -MVEXPAND_MULTILINE_COMMENT=105 -MVEXPAND_WS=106 -INFO=107 -SHOW_LINE_COMMENT=108 -SHOW_MULTILINE_COMMENT=109 -SHOW_WS=110 -SETTING=111 -SETTING_LINE_COMMENT=112 -SETTTING_MULTILINE_COMMENT=113 -SETTING_WS=114 -LOOKUP_LINE_COMMENT=115 -LOOKUP_MULTILINE_COMMENT=116 -LOOKUP_WS=117 -LOOKUP_FIELD_LINE_COMMENT=118 -LOOKUP_FIELD_MULTILINE_COMMENT=119 -LOOKUP_FIELD_WS=120 +UNKNOWN_CMD=24 +LINE_COMMENT=25 +MULTILINE_COMMENT=26 +WS=27 +PIPE=28 +QUOTED_STRING=29 +INTEGER_LITERAL=30 +DECIMAL_LITERAL=31 +BY=32 +AND=33 +ASC=34 +ASSIGN=35 +CAST_OP=36 +COLON=37 +COMMA=38 +DESC=39 +DOT=40 +FALSE=41 +FIRST=42 +IN=43 +IS=44 +LAST=45 +LIKE=46 +LP=47 +NOT=48 +NULL=49 +NULLS=50 +OR=51 +PARAM=52 +RLIKE=53 +RP=54 +TRUE=55 +EQ=56 +CIEQ=57 +NEQ=58 +LT=59 +LTE=60 +GT=61 +GTE=62 +PLUS=63 +MINUS=64 +ASTERISK=65 +SLASH=66 +PERCENT=67 +LEFT_BRACES=68 +RIGHT_BRACES=69 +NAMED_OR_POSITIONAL_PARAM=70 +OPENING_BRACKET=71 +CLOSING_BRACKET=72 +UNQUOTED_IDENTIFIER=73 +QUOTED_IDENTIFIER=74 +EXPR_LINE_COMMENT=75 +EXPR_MULTILINE_COMMENT=76 +EXPR_WS=77 +EXPLAIN_WS=78 +EXPLAIN_LINE_COMMENT=79 +EXPLAIN_MULTILINE_COMMENT=80 +METADATA=81 +UNQUOTED_SOURCE=82 +FROM_LINE_COMMENT=83 +FROM_MULTILINE_COMMENT=84 +FROM_WS=85 +ID_PATTERN=86 +PROJECT_LINE_COMMENT=87 +PROJECT_MULTILINE_COMMENT=88 +PROJECT_WS=89 +AS=90 +RENAME_LINE_COMMENT=91 +RENAME_MULTILINE_COMMENT=92 +RENAME_WS=93 +ON=94 +WITH=95 +ENRICH_POLICY_NAME=96 +ENRICH_LINE_COMMENT=97 +ENRICH_MULTILINE_COMMENT=98 +ENRICH_WS=99 +ENRICH_FIELD_LINE_COMMENT=100 +ENRICH_FIELD_MULTILINE_COMMENT=101 +ENRICH_FIELD_WS=102 +MVEXPAND_LINE_COMMENT=103 +MVEXPAND_MULTILINE_COMMENT=104 +MVEXPAND_WS=105 +INFO=106 +SHOW_LINE_COMMENT=107 +SHOW_MULTILINE_COMMENT=108 +SHOW_WS=109 +SETTING=110 +SETTING_LINE_COMMENT=111 +SETTTING_MULTILINE_COMMENT=112 +SETTING_WS=113 +LOOKUP_LINE_COMMENT=114 +LOOKUP_MULTILINE_COMMENT=115 +LOOKUP_WS=116 +LOOKUP_FIELD_LINE_COMMENT=117 +LOOKUP_FIELD_MULTILINE_COMMENT=118 +LOOKUP_FIELD_WS=119 +JOIN=120 USING=121 JOIN_LINE_COMMENT=122 JOIN_MULTILINE_COMMENT=123 @@ -144,49 +144,51 @@ CLOSING_METRICS_WS=130 'sort'=14 'stats'=15 'where'=16 -'|'=29 -'by'=33 -'and'=34 -'asc'=35 -'='=36 -'::'=37 -':'=38 -','=39 -'desc'=40 -'.'=41 -'false'=42 -'first'=43 -'in'=44 -'is'=45 -'last'=46 -'like'=47 -'('=48 -'not'=49 -'null'=50 -'nulls'=51 -'or'=52 -'?'=53 -'rlike'=54 -')'=55 -'true'=56 -'=='=57 -'=~'=58 -'!='=59 -'<'=60 -'<='=61 -'>'=62 -'>='=63 -'+'=64 -'-'=65 -'*'=66 -'/'=67 -'%'=68 -'{'=69 -'}'=70 -']'=73 -'metadata'=82 -'as'=91 -'on'=95 -'with'=96 -'info'=107 +'lookup'=17 +'|'=28 +'by'=32 +'and'=33 +'asc'=34 +'='=35 +'::'=36 +':'=37 +','=38 +'desc'=39 +'.'=40 +'false'=41 +'first'=42 +'in'=43 +'is'=44 +'last'=45 +'like'=46 +'('=47 +'not'=48 +'null'=49 +'nulls'=50 +'or'=51 +'?'=52 +'rlike'=53 +')'=54 +'true'=55 +'=='=56 +'=~'=57 +'!='=58 +'<'=59 +'<='=60 +'>'=61 +'>='=62 +'+'=63 +'-'=64 +'*'=65 +'/'=66 +'%'=67 +'{'=68 +'}'=69 +']'=72 +'metadata'=81 +'as'=90 +'on'=94 +'with'=95 +'info'=106 +'join'=120 'USING'=121 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index b57a22c494a2a..8e6a0c8105923 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -20,6 +20,8 @@ import java.util.Locale; import java.util.Set; +import static org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG; + /** * A {@link Set} of "capabilities" supported by the {@link RestEsqlQueryAction} * and {@link RestEsqlAsyncQueryAction} APIs. These are exposed over the @@ -187,9 +189,12 @@ public enum Cap { */ ST_DISTANCE, - /** Support for function {@code ST_EXTENT}. */ + /** Support for function {@code ST_EXTENT_AGG}. */ ST_EXTENT_AGG, + /** Optimization of ST_EXTENT_AGG with doc-values as IntBlock. */ + ST_EXTENT_AGG_DOCVALUES, + /** * Fix determination of CRS types in spatial functions when folding. */ @@ -393,6 +398,15 @@ public enum Cap { * support date diff function on date nanos type, and mixed nanos/millis */ DATE_NANOS_DATE_DIFF(), + /** + * Indicates that https://github.com/elastic/elasticsearch/issues/125439 (incorrect lucene push down for date nanos) is fixed + */ + FIX_DATE_NANOS_LUCENE_PUSHDOWN_BUG(), + /** + * Fixes a bug where dates are incorrectly formatted if a where clause compares nanoseconds to both milliseconds and nanoseconds, + * e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")} + */ + FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(), /** * DATE_PARSE supports reading timezones */ @@ -551,7 +565,7 @@ public enum Cap { /** * Support simplified syntax for named parameters for field and function names. */ - NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX(Build.current().isSnapshot()), + NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX(), /** * Fix pushdown of LIMIT past MV_EXPAND @@ -577,7 +591,7 @@ public enum Cap { /** * LOOKUP JOIN */ - JOIN_LOOKUP_V12(Build.current().isSnapshot()), + JOIN_LOOKUP_V12, /** * LOOKUP JOIN with TEXT fields on the right (right side of the join) (#119473) @@ -662,7 +676,42 @@ public enum Cap { /** * Support for aggregate_metric_double type */ - AGGREGATE_METRIC_DOUBLE; + AGGREGATE_METRIC_DOUBLE(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG.isEnabled()), + + /** + * Fix for https://github.com/elastic/elasticsearch/issues/120817 + * and https://github.com/elastic/elasticsearch/issues/120803 + * Support for queries that have multiple SORTs that cannot become TopN + */ + REMOVE_REDUNDANT_SORT, + + /** + * Lucene query pushdown to StartsWith and EndsWith functions. + * This capability was created to avoid receiving wrong warnings from old nodes in mixed clusters + */ + STARTS_WITH_ENDS_WITH_LUCENE_PUSHDOWN, + + /** + * Allow mixed numeric types in conditional functions - case, greatest and least + */ + MIXED_NUMERIC_TYPES_IN_CASE_GREATEST_LEAST, + + /** + * Make numberOfChannels consistent with layout in DefaultLayout by removing duplicated ChannelSet. + */ + MAKE_NUMBER_OF_CHANNELS_CONSISTENT_WITH_LAYOUT, + + /** + * Supercedes {@link Cap#MAKE_NUMBER_OF_CHANNELS_CONSISTENT_WITH_LAYOUT}. + */ + FIX_REPLACE_MISSING_FIELD_WITH_NULL_DUPLICATE_NAME_ID_IN_LAYOUT, + + /** + * When creating constant null blocks in {@link org.elasticsearch.compute.lucene.ValuesSourceReaderOperator}, we also handed off + * the ownership of that block - but didn't account for the fact that the caller might close it, leading to double releases + * in some union type queries. C.f. https://github.com/elastic/elasticsearch/issues/125850 + */ + FIX_DOUBLY_RELEASED_NULL_BLOCKS_IN_VALUESOURCEREADER; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java index d8904288523a7..e77d7b41aaca6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java @@ -171,8 +171,7 @@ private static QueryParams parseParams(XContentParser p) throws IOException { String paramName = entry.getKey(); checkParamNameValidity(paramName, errors, loc); - if (EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - && entry.getValue() instanceof Map value) {// parameter specified as a key:value pair + if (entry.getValue() instanceof Map value) {// parameter specified as a key:value pair checkParamValueSize(paramName, value, loc, errors); for (Object keyName : value.keySet()) { classification = getParamClassification(keyName.toString(), errors, loc); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 1351b5ce51f44..12a198f76857b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -51,6 +51,9 @@ import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -1208,7 +1211,7 @@ private static Expression processIn(In in) { } private static boolean canCastMixedNumericTypes(org.elasticsearch.xpack.esql.core.expression.function.Function f) { - return f instanceof Coalesce; + return f instanceof Coalesce || f instanceof Case || f instanceof Greatest || f instanceof Least; } private static boolean canCastNumeric(DataType from, DataType to) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java index f68f9f2487884..248c151bcf948 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java @@ -39,7 +39,7 @@ protected SpatialAggregateFunction(StreamInput in, FieldExtractPreference fieldE this.fieldExtractPreference = fieldExtractPreference; } - public abstract SpatialAggregateFunction withDocValues(); + public abstract SpatialAggregateFunction withFieldExtractPreference(FieldExtractPreference preference); @Override public boolean licenseCheck(XPackLicenseState state) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java index ab0eb52cbe060..db3d95fb9059c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java @@ -71,8 +71,8 @@ public SpatialCentroid withFilter(Expression filter) { } @Override - public SpatialCentroid withDocValues() { - return new SpatialCentroid(source(), field(), filter(), FieldExtractPreference.DOC_VALUES); + public SpatialCentroid withFieldExtractPreference(FieldExtractPreference preference) { + return new SpatialCentroid(source(), field(), filter(), preference); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java index a9922eef36746..90c9fdd500899 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtent.java @@ -11,10 +11,12 @@ import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianPointDocValuesAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianPointSourceValuesAggregatorFunctionSupplier; -import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianShapeAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoPointDocValuesAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoPointSourceValuesAggregatorFunctionSupplier; -import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoShapeAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.spatial.SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier; import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -75,8 +77,8 @@ public SpatialExtent withFilter(Expression filter) { } @Override - public org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialExtent withDocValues() { - return new SpatialExtent(source(), field(), filter(), FieldExtractPreference.DOC_VALUES); + public SpatialExtent withFieldExtractPreference(FieldExtractPreference preference) { + return new SpatialExtent(source(), field(), filter(), preference); } @Override @@ -101,7 +103,8 @@ public SpatialExtent replaceChildren(List newChildren) { @Override public AggregatorFunctionSupplier supplier(List inputChannels) { - return switch (field().dataType()) { + DataType type = field().dataType(); + return switch (type) { case GEO_POINT -> switch (fieldExtractPreference) { case DOC_VALUES -> new SpatialExtentGeoPointDocValuesAggregatorFunctionSupplier(inputChannels); case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels); @@ -110,10 +113,17 @@ public AggregatorFunctionSupplier supplier(List inputChannels) { case DOC_VALUES -> new SpatialExtentCartesianPointDocValuesAggregatorFunctionSupplier(inputChannels); case NONE, EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels); }; - // Shapes don't differentiate between source and doc values. - case GEO_SHAPE -> new SpatialExtentGeoShapeAggregatorFunctionSupplier(inputChannels); - case CARTESIAN_SHAPE -> new SpatialExtentCartesianShapeAggregatorFunctionSupplier(inputChannels); - default -> throw EsqlIllegalArgumentException.illegalDataType(field().dataType()); + case GEO_SHAPE -> switch (fieldExtractPreference) { + case EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentGeoShapeDocValuesAggregatorFunctionSupplier(inputChannels); + case NONE -> new SpatialExtentGeoShapeSourceValuesAggregatorFunctionSupplier(inputChannels); + case DOC_VALUES -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); + }; + case CARTESIAN_SHAPE -> switch (fieldExtractPreference) { + case EXTRACT_SPATIAL_BOUNDS -> new SpatialExtentCartesianShapeDocValuesAggregatorFunctionSupplier(inputChannels); + case NONE -> new SpatialExtentCartesianShapeSourceValuesAggregatorFunctionSupplier(inputChannels); + case DOC_VALUES -> throw new EsqlIllegalArgumentException("Illegal field extract preference: " + fieldExtractPreference); + }; + default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index 3223e96da7136..27e31235b0b85 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -177,13 +177,14 @@ public Match( name = "analyzer", type = "keyword", valueHint = { "standard" }, - description = "Analyzer used to convert the text in the query value into token." + description = "Analyzer used to convert the text in the query value into token. Defaults to the index-time analyzer" + + " mapped for the field. If no analyzer is mapped, the index’s default analyzer is used." ), @MapParam.MapParamEntry( name = "auto_generate_synonyms_phrase_query", type = "boolean", valueHint = { "true", "false" }, - description = "If true, match phrase queries are automatically created for multi-term synonyms." + description = "If true, match phrase queries are automatically created for multi-term synonyms. Defaults to true." ), @MapParam.MapParamEntry( name = "fuzziness", @@ -195,13 +196,14 @@ public Match( name = "boost", type = "float", valueHint = { "2.5" }, - description = "Floating point number used to decrease or increase the relevance scores of the query." + description = "Floating point number used to decrease or increase the relevance scores of the query. Defaults to 1.0." ), @MapParam.MapParamEntry( name = "fuzzy_transpositions", type = "boolean", valueHint = { "true", "false" }, - description = "If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba)." + description = "If true, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). " + + "Defaults to true." ), @MapParam.MapParamEntry( name = "fuzzy_rewrite", @@ -213,19 +215,22 @@ public Match( "top_terms_blended_freqs_N", "top_terms_boost_N", "top_terms_N" }, - description = "Method used to rewrite the query. See the rewrite parameter for valid values and more information." + description = "Method used to rewrite the query. See the rewrite parameter for valid values and more information. " + + "If the fuzziness parameter is not 0, the match query uses a fuzzy_rewrite method of " + + "top_terms_blended_freqs_${max_expansions} by default." ), @MapParam.MapParamEntry( name = "lenient", type = "boolean", valueHint = { "true", "false" }, - description = "If false, format-based errors, such as providing a text query value for a numeric field, are returned." + description = "If false, format-based errors, such as providing a text query value for a numeric field, are returned. " + + "Defaults to false." ), @MapParam.MapParamEntry( name = "max_expansions", type = "integer", valueHint = { "50" }, - description = "Maximum number of terms to which the query will expand." + description = "Maximum number of terms to which the query will expand. Defaults to 50." ), @MapParam.MapParamEntry( name = "minimum_should_match", @@ -237,21 +242,22 @@ public Match( name = "operator", type = "keyword", valueHint = { "AND", "OR" }, - description = "Boolean logic used to interpret text in the query value." + description = "Boolean logic used to interpret text in the query value. Defaults to OR." ), @MapParam.MapParamEntry( name = "prefix_length", type = "integer", valueHint = { "1" }, - description = "Number of beginning characters left unchanged for fuzzy matching." + description = "Number of beginning characters left unchanged for fuzzy matching. Defaults to 0." ), @MapParam.MapParamEntry( name = "zero_terms_query", type = "keyword", valueHint = { "none", "all" }, - description = "Number of beginning characters left unchanged for fuzzy matching." + description = "Indicates whether all documents or none are returned if the analyzer removes all tokens, such as " + + "when using a stop filter. Defaults to none." ) }, - description = "Match additional options as <>." + description = "(Optional) Match additional options as <>." + " See <> for more information.", optional = true ) Expression options diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index e97e65a3e60fc..b3d50d7b572fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -7,13 +7,20 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; +import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -22,6 +29,8 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import java.io.IOException; import java.util.Arrays; @@ -31,7 +40,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; -public class EndsWith extends EsqlScalarFunction { +public class EndsWith extends EsqlScalarFunction implements TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "EndsWith", EndsWith::new); private final Expression str; @@ -129,6 +138,27 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new EndsWithEvaluator.Factory(source(), toEvaluator.apply(str), toEvaluator.apply(suffix)); } + @Override + public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(str) && suffix.foldable(); + } + + @Override + public Query asQuery(TranslatorHandler handler) { + LucenePushdownPredicates.checkIsPushableAttribute(str); + var fieldName = handler.nameOf(str instanceof FieldAttribute fa ? fa.exactAttribute() : str); + + // TODO: Get the real FoldContext here + var wildcardQuery = "*" + QueryParser.escape(BytesRefs.toString(suffix.fold(FoldContext.small()))); + + return new WildcardQuery(source(), fieldName, wildcardQuery); + } + + @Override + public Expression singleValueField() { + return str; + } + Expression str() { return str; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index 2256ec2179adf..9ab552576dbbb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -7,13 +7,20 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; +import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -22,6 +29,8 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import java.io.IOException; import java.util.Arrays; @@ -31,7 +40,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; -public class StartsWith extends EsqlScalarFunction { +public class StartsWith extends EsqlScalarFunction implements TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "StartsWith", @@ -126,6 +135,27 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StartsWithEvaluator.Factory(source(), toEvaluator.apply(str), toEvaluator.apply(prefix)); } + @Override + public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(str) && prefix.foldable(); + } + + @Override + public Query asQuery(TranslatorHandler handler) { + LucenePushdownPredicates.checkIsPushableAttribute(str); + var fieldName = handler.nameOf(str instanceof FieldAttribute fa ? fa.exactAttribute() : str); + + // TODO: Get the real FoldContext here + var wildcardQuery = QueryParser.escape(BytesRefs.toString(prefix.fold(FoldContext.small()))) + "*"; + + return new WildcardQuery(source(), fieldName, wildcardQuery); + } + + @Override + public Expression singleValueField() { + return str; + } + Expression str() { return str; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index 4da10c5ec7b8c..d42c46863952e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -8,6 +8,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; @@ -31,18 +33,21 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; // BETWEEN or range - is a mix of gt(e) AND lt(e) public class Range extends ScalarFunction implements TranslationAware.SingleValueTranslationAware { + private static final Logger logger = LogManager.getLogger(Range.class); private final Expression value, lower, upper; private final boolean includeLower, includeUpper; @@ -210,12 +215,26 @@ private RangeQuery translate(TranslatorHandler handler) { String format = null; DataType dataType = value.dataType(); - if (DataType.isDateTime(dataType) && DataType.isDateTime(lower.dataType()) && DataType.isDateTime(upper.dataType())) { - l = dateTimeToString((Long) l); - u = dateTimeToString((Long) u); + logger.trace( + "Translating Range into lucene query. dataType is [{}] upper is [{}<{}>] lower is [{}<{}>]", + dataType, + lower, + lower.dataType(), + upper, + upper.dataType() + ); + if (dataType == DataType.DATETIME) { + l = dateWithTypeToString((Long) l, lower.dataType()); + u = dateWithTypeToString((Long) u, upper.dataType()); format = DEFAULT_DATE_TIME_FORMATTER.pattern(); } + if (dataType == DATE_NANOS) { + l = dateWithTypeToString((Long) l, lower.dataType()); + u = dateWithTypeToString((Long) u, upper.dataType()); + format = DEFAULT_DATE_NANOS_FORMATTER.pattern(); + } + if (dataType == IP) { if (l instanceof BytesRef bytesRef) { l = ipToString(bytesRef); @@ -244,6 +263,7 @@ private RangeQuery translate(TranslatorHandler handler) { u = unsignedLongAsNumber(ul); } } + logger.trace("Building range query with format string [{}]", format); return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index fc69f4dc19d72..d9d5aa985ded1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -13,6 +13,8 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; @@ -50,14 +52,17 @@ import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.HOUR_MINUTE_SECOND; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; @@ -66,6 +71,8 @@ public abstract class EsqlBinaryComparison extends BinaryComparison EvaluatorMapper, TranslationAware.SingleValueTranslationAware { + private static final Logger logger = LogManager.getLogger(EsqlBinaryComparison.class); + private final Map evaluatorMap; private final BinaryComparisonOperation functionType; @@ -375,6 +382,16 @@ private Query translate(TranslatorHandler handler) { String format = null; boolean isDateLiteralComparison = false; + logger.trace( + "Translating binary comparison with right: [{}<{}>], left: [{}<{}>], attribute: [{}<{}>]", + right(), + right().dataType(), + left(), + left().dataType(), + attribute, + attribute.dataType() + ); + // TODO: This type coersion layer is copied directly from the QL counterpart code. It's probably not necessary or desireable // in the ESQL version. We should instead do the type conversions using our casting functions. // for a date constant comparison, we need to use a format for the date, to make sure that the format is the same @@ -382,7 +399,12 @@ private Query translate(TranslatorHandler handler) { if (value instanceof ZonedDateTime || value instanceof OffsetTime) { DateFormatter formatter; if (value instanceof ZonedDateTime) { - formatter = DEFAULT_DATE_TIME_FORMATTER; + // NB: we check the data type of right here because value is the RHS value + formatter = switch (right().dataType()) { + case DATETIME -> DEFAULT_DATE_TIME_FORMATTER; + case DATE_NANOS -> DEFAULT_DATE_NANOS_FORMATTER; + default -> throw new EsqlIllegalArgumentException("Found date value in non-date type comparison"); + }; // RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime instance // which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime // Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format as well. @@ -408,10 +430,14 @@ private Query translate(TranslatorHandler handler) { } ZoneId zoneId = null; - if (DataType.isDateTime(attribute.dataType())) { + if (attribute.dataType() == DATETIME) { zoneId = zoneId(); - value = dateTimeToString((Long) value); + value = dateWithTypeToString((Long) value, right().dataType()); format = DEFAULT_DATE_TIME_FORMATTER.pattern(); + } else if (attribute.dataType() == DATE_NANOS) { + zoneId = zoneId(); + value = dateWithTypeToString((Long) value, right().dataType()); + format = DEFAULT_DATE_NANOS_FORMATTER.pattern(); } if (this instanceof GreaterThan) { return new RangeQuery(source(), name, value, false, null, false, format, zoneId); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index 709a82733c57e..bcbcbda33f9ad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -15,6 +15,8 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -115,6 +117,7 @@ */ public class In extends EsqlScalarFunction implements TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "In", In::new); + private static final Logger logger = LogManager.getLogger(In.class); private final Expression value; private final List list; @@ -468,6 +471,7 @@ public Query asQuery(TranslatorHandler handler) { } private Query translate(TranslatorHandler handler) { + logger.trace("Attempting to generate lucene query for IN expression"); TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(value()); Set terms = new LinkedHashSet<>(); @@ -501,7 +505,7 @@ private Query translate(TranslatorHandler handler) { } private static boolean needsTypeSpecificValueHandling(DataType fieldType) { - return DataType.isDateTime(fieldType) || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG; + return fieldType == DATETIME || fieldType == DATE_NANOS || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG; } private static Query or(Source source, Query left, Query right) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 36150083daec0..bc32945d73eb5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -10,7 +10,6 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.AddDefaultTopN; import org.elasticsearch.xpack.esql.optimizer.rules.logical.BooleanFunctionEqualsElimination; import org.elasticsearch.xpack.esql.optimizer.rules.logical.BooleanSimplification; import org.elasticsearch.xpack.esql.optimizer.rules.logical.CombineBinaryComparisons; @@ -32,7 +31,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneEmptyPlans; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneFilters; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneLiteralsInOrderBy; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneOrderByBeforeStats; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneRedundantOrderBy; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneRedundantSortClauses; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownAndCombineFilters; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownAndCombineLimits; @@ -116,10 +115,9 @@ protected List> batches() { protected static List> rules() { var skip = new Batch<>("Skip Compute", new SkipQueryOnLimitZero()); - var defaultTopN = new Batch<>("Add default TopN", new AddDefaultTopN()); var label = new Batch<>("Set as Optimized", Limiter.ONCE, new SetAsOptimized()); - return asList(substitutions(), operators(), skip, cleanup(), defaultTopN, label); + return asList(substitutions(), operators(), skip, cleanup(), label); } protected static Batch substitutions() { @@ -189,7 +187,7 @@ protected static Batch operators() { new PushDownRegexExtract(), new PushDownEnrich(), new PushDownAndCombineOrderBy(), - new PruneOrderByBeforeStats(), + new PruneRedundantOrderBy(), new PruneRedundantSortClauses() ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java index 94248ce2ecd0a..c474c48d6d96b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java @@ -27,6 +27,9 @@ public Failures verify(LogicalPlan plan) { PlanConsistencyChecker.checkPlan(p, dependencyFailures); if (failures.hasFailures() == false) { + if (p instanceof PostOptimizationVerificationAware pova) { + pova.postOptimizationVerification(failures); + } p.forEachExpression(ex -> { if (ex instanceof PostOptimizationVerificationAware va) { va.postOptimizationVerification(failures); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/AddDefaultTopN.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/AddDefaultTopN.java deleted file mode 100644 index ef091686a4b38..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/AddDefaultTopN.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.optimizer.rules.logical; - -import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; -import org.elasticsearch.xpack.esql.plan.logical.EsRelation; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.plan.logical.OrderBy; -import org.elasticsearch.xpack.esql.plan.logical.TopN; -import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; - -/** - * This adds an explicit TopN node to a plan that only has an OrderBy right before Lucene. - * To date, the only known use case that "needs" this is a query of the form - * from test - * | sort emp_no - * | mv_expand first_name - * | rename first_name AS x - * | where x LIKE "*a*" - * | limit 15 - *

- * or - *

- * from test - * | sort emp_no - * | mv_expand first_name - * | sort first_name - * | limit 15 - *

- * {@link PushDownAndCombineLimits} will copy the "limit 15" after "sort emp_no" if there is no filter on the expanded values - * OR if there is no sort between "limit" and "mv_expand". - * But, since this type of query has such a filter, the "sort emp_no" will have no limit when it reaches the current rule. - */ -public final class AddDefaultTopN extends OptimizerRules.ParameterizedOptimizerRule { - public AddDefaultTopN() { - super(OptimizerRules.TransformDirection.DOWN); - } - - @Override - protected LogicalPlan rule(LogicalPlan plan, LogicalOptimizerContext context) { - if (plan instanceof UnaryPlan unary && unary.child() instanceof OrderBy order && order.child() instanceof EsRelation relation) { - var limit = new Literal(plan.source(), context.configuration().resultTruncationMaxSize(), DataType.INTEGER); - return unary.replaceChild(new TopN(plan.source(), relation, order.order(), limit)); - } - return plan; - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneOrderByBeforeStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneOrderByBeforeStats.java deleted file mode 100644 index 24fb8971487d5..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneOrderByBeforeStats.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.optimizer.rules.logical; - -import org.elasticsearch.xpack.esql.plan.logical.Aggregate; -import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.logical.Eval; -import org.elasticsearch.xpack.esql.plan.logical.Filter; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.plan.logical.OrderBy; -import org.elasticsearch.xpack.esql.plan.logical.Project; -import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; -import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; - -public final class PruneOrderByBeforeStats extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(Aggregate agg) { - OrderBy order = findPullableOrderBy(agg.child()); - - LogicalPlan p = agg; - if (order != null) { - p = agg.transformDown(OrderBy.class, o -> o == order ? order.child() : o); - } - return p; - } - - private static OrderBy findPullableOrderBy(LogicalPlan plan) { - OrderBy pullable = null; - if (plan instanceof OrderBy o) { - pullable = o; - } else if (plan instanceof Eval - || plan instanceof Filter - || plan instanceof Project - || plan instanceof RegexExtract - || plan instanceof Enrich) { - pullable = findPullableOrderBy(((UnaryPlan) plan).child()); - } - return pullable; - } - -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneRedundantOrderBy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneRedundantOrderBy.java new file mode 100644 index 0000000000000..2495f72864d1c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneRedundantOrderBy.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.plan.logical.SortAgnostic; +import org.elasticsearch.xpack.esql.plan.logical.TopN; +import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; + +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.Set; + +/** + * SORT cannot be executed without a LIMIT, as ES|QL doesn't support unbounded sort (yet). + *

+ * The planner tries to push down LIMIT and transform all the unbounded sorts into a TopN. + * In some cases it's not possible though, eg. + *

+ * from test | sort x | lookup join lookup on x | sort y + *

+ * from test | sort x | mv_expand x | sort y + *

+ * "sort y" will become a TopN due to the addition of the default Limit, but "sort x" will remain unbounded, + * so the query could not be executed. + *

+ * In most cases though, following commands can make the previous SORTs redundant, + * because it will re-sort previously sorted results (eg. if there is another SORT) + * or because the order will be scrambled by another command (eg. a STATS) + *

+ * This rule finds and prunes redundant SORTs, attempting to make the plan executable. + */ +public class PruneRedundantOrderBy extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(LogicalPlan plan) { + if (plan instanceof OrderBy || plan instanceof TopN || plan instanceof Aggregate) { + Set redundant = findRedundantSort(((UnaryPlan) plan).child()); + if (redundant.isEmpty()) { + return plan; + } + return plan.transformDown(p -> redundant.contains(p) ? ((UnaryPlan) p).child() : p); + } else { + return plan; + } + } + + /** + * breadth-first recursion to find redundant SORTs in the children tree. + * Returns an identity set (we need to compare and prune the exact instances) + */ + private Set findRedundantSort(LogicalPlan plan) { + Set result = Collections.newSetFromMap(new IdentityHashMap<>()); + + Deque toCheck = new ArrayDeque<>(); + toCheck.push(plan); + + while (true) { + if (toCheck.isEmpty()) { + return result; + } + LogicalPlan p = toCheck.pop(); + if (p instanceof OrderBy ob) { + result.add(ob); + toCheck.push(ob.child()); + } else if (p instanceof SortAgnostic) { + for (LogicalPlan child : p.children()) { + toCheck.push(child); + } + } + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java index e41e500aad110..5efef5e4b7c9a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceMissingFieldWithNull.java @@ -10,13 +10,13 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; -import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -25,14 +25,12 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.TopN; -import org.elasticsearch.xpack.esql.plan.logical.join.Join; -import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.rule.ParameterizedRule; -import org.elasticsearch.xpack.esql.stats.SearchStats; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Predicate; /** * Look for any fields used in the plan that are missing locally and replace them with null. @@ -42,79 +40,85 @@ public class ReplaceMissingFieldWithNull extends ParameterizedRule { + // Looking only for indices in LOOKUP mode is correct: during parsing, we assign the expected mode and even if a lookup index + // is used in the FROM command, it will not be marked with LOOKUP mode there - but STANDARD. + // It seems like we could instead just look for JOINs and walk down their right hand side to find lookup fields - but this does + // not work as this rule also gets called just on the right hand side of a JOIN, which means that we don't always know that + // we're inside the right (or left) branch of a JOIN node. (See PlannerUtils.localPlan - this looks for FragmentExecs and + // performs local logical optimization of the fragments; the right hand side of a LookupJoinExec can be a FragmentExec.) if (esRelation.indexMode() == IndexMode.LOOKUP) { lookupFields.addAll(esRelation.output()); } }); - return plan.transformUp(p -> missingToNull(p, localLogicalOptimizerContext.searchStats(), lookupFields)); - } - - private LogicalPlan missingToNull(LogicalPlan plan, SearchStats stats, AttributeSet lookupFields) { - if (plan instanceof EsRelation || plan instanceof LocalRelation) { - return plan; - } + // Do not use the attribute name, this can deviate from the field name for union types; use fieldName() instead. + // Also retain fields from lookup indices because we do not have stats for these. + Predicate shouldBeRetained = f -> (localLogicalOptimizerContext.searchStats().exists(f.fieldName()) + || lookupFields.contains(f)); - if (plan instanceof Aggregate a) { - // don't do anything (for now) - return a; - } - // keep the aliased name - else if (plan instanceof Project project) { - var projections = project.projections(); - List newProjections = new ArrayList<>(projections.size()); - Map nullLiteral = Maps.newLinkedHashMapWithExpectedSize(DataType.types().size()); - AttributeSet joinAttributes = joinAttributes(project); + return plan.transformUp(p -> missingToNull(p, shouldBeRetained)); + } - for (NamedExpression projection : projections) { - // Do not use the attribute name, this can deviate from the field name for union types. - if (projection instanceof FieldAttribute f && stats.exists(f.fieldName()) == false && joinAttributes.contains(f) == false) { - // TODO: Should do a searchStats lookup for join attributes instead of just ignoring them here - // See TransportSearchShardsAction + private LogicalPlan missingToNull(LogicalPlan plan, Predicate shouldBeRetained) { + if (plan instanceof EsRelation relation) { + // Remove missing fields from the EsRelation because this is not where we will obtain them from; replace them by an Eval right + // after, instead. This allows us to safely re-use the attribute ids of the corresponding FieldAttributes. + // This means that an EsRelation[field1, field2, field3] where field1 and field 3 are missing will be replaced by + // Project[field1, field2, field3] <- keeps the ordering intact + // \_Eval[field1 = null, field3 = null] + // \_EsRelation[field2] + List relationOutput = relation.output(); + Map nullLiterals = Maps.newLinkedHashMapWithExpectedSize(DataType.types().size()); + List newProjections = new ArrayList<>(relationOutput.size()); + for (int i = 0, size = relationOutput.size(); i < size; i++) { + Attribute attr = relationOutput.get(i); + NamedExpression projection; + if (attr instanceof FieldAttribute f && (shouldBeRetained.test(f) == false)) { DataType dt = f.dataType(); - Alias nullAlias = nullLiteral.get(f.dataType()); + Alias nullAlias = nullLiterals.get(dt); // save the first field as null (per datatype) if (nullAlias == null) { + // Keep the same id so downstream query plans don't need updating + // NOTE: THIS IS BRITTLE AND CAN LEAD TO BUGS. + // In case some optimizer rule or so inserts a plan node that requires the field BEFORE the Eval that we're adding + // on top of the EsRelation, this can trigger a field extraction in the physical optimizer phase, causing wrong + // layouts due to a duplicate name id. + // If someone reaches here AGAIN when debugging e.g. ClassCastExceptions NPEs from wrong layouts, we should probably + // give up on this approach and instead insert EvalExecs in InsertFieldExtraction. Alias alias = new Alias(f.source(), f.name(), Literal.of(f, null), f.id()); - nullLiteral.put(dt, alias); + nullLiterals.put(dt, alias); projection = alias.toAttribute(); } - // otherwise point to it + // otherwise point to it since this avoids creating field copies else { - // since avoids creating field copies projection = new Alias(f.source(), f.name(), nullAlias.toAttribute(), f.id()); } + } else { + projection = attr; } - newProjections.add(projection); } - // add the first found field as null - if (nullLiteral.size() > 0) { - plan = new Eval(project.source(), project.child(), new ArrayList<>(nullLiteral.values())); - plan = new Project(project.source(), plan, newProjections); + + if (nullLiterals.size() == 0) { + return plan; } - } else if (plan instanceof Eval + + Eval eval = new Eval(plan.source(), relation, new ArrayList<>(nullLiterals.values())); + // This projection is redundant if there's another projection downstream (and no commands depend on the order until we hit it). + return new Project(plan.source(), eval, newProjections); + } + + if (plan instanceof Eval || plan instanceof Filter || plan instanceof OrderBy || plan instanceof RegexExtract || plan instanceof TopN) { - plan = plan.transformExpressionsOnlyUp( - FieldAttribute.class, - // Do not use the attribute name, this can deviate from the field name for union types. - // Also skip fields from lookup indices because we do not have stats for these. - // TODO: We do have stats for lookup indices in case they are being used in the FROM clause; this can be refined. - f -> stats.exists(f.fieldName()) || lookupFields.contains(f) ? f : Literal.of(f, null) - ); - } + return plan.transformExpressionsOnlyUp(FieldAttribute.class, f -> shouldBeRetained.test(f) ? f : Literal.of(f, null)); + } return plan; } - - private AttributeSet joinAttributes(Project project) { - var attributes = new AttributeSet(); - project.forEachDown(Join.class, j -> j.right().forEachDown(EsRelation.class, p -> attributes.addAll(p.output()))); - return attributes; - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 2f28b1a0e41ba..f902f261e7dc9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -101,7 +101,8 @@ private static PhysicalPlan rewrite( if (newPushable.size() > 0) { // update the executable with pushable conditions Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable)); QueryBuilder planQuery = queryDSL.asBuilder(); - var query = Queries.combine(Queries.Clause.FILTER, asList(queryExec.query(), planQuery)); + Queries.Clause combiningQueryClauseType = queryExec.hasScoring() ? Queries.Clause.MUST : Queries.Clause.FILTER; + var query = Queries.combine(combiningQueryClauseType, asList(queryExec.query(), planQuery)); queryExec = new EsQueryExec( queryExec.source(), queryExec.indexPattern(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java index 4f3358c539b05..4730f561348c9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java @@ -32,27 +32,36 @@ protected PhysicalPlan rule(EsSourceExec plan) { var docId = new FieldAttribute(plan.source(), EsQueryExec.DOC_ID_FIELD.getName(), EsQueryExec.DOC_ID_FIELD); final List attributes = new ArrayList<>(); attributes.add(docId); - if (plan.indexMode() == IndexMode.TIME_SERIES) { - Attribute tsid = null, timestamp = null; - for (Attribute attr : plan.output()) { - String name = attr.name(); - if (name.equals(MetadataAttribute.TSID_FIELD)) { + + var outputIterator = plan.output().iterator(); + var isTimeSeries = plan.indexMode() == IndexMode.TIME_SERIES; + var keepIterating = true; + Attribute tsid = null, timestamp = null, score = null; + + while (keepIterating && outputIterator.hasNext()) { + Attribute attr = outputIterator.next(); + if (attr instanceof MetadataAttribute ma) { + if (ma.name().equals(MetadataAttribute.SCORE)) { + score = attr; + } else if (isTimeSeries && ma.name().equals(MetadataAttribute.TSID_FIELD)) { tsid = attr; - } else if (name.equals(MetadataAttribute.TIMESTAMP_FIELD)) { - timestamp = attr; } + } else if (attr.name().equals(MetadataAttribute.TIMESTAMP_FIELD)) { + timestamp = attr; } + keepIterating = score == null || (isTimeSeries && (tsid == null || timestamp == null)); + } + if (isTimeSeries) { if (tsid == null || timestamp == null) { throw new IllegalStateException("_tsid or @timestamp are missing from the time-series source"); } attributes.add(tsid); attributes.add(timestamp); } - plan.output().forEach(attr -> { - if (attr instanceof MetadataAttribute ma && ma.name().equals(MetadataAttribute.SCORE)) { - attributes.add(ma); - } - }); + if (score != null) { + attributes.add(score); + } + return new EsQueryExec(plan.source(), plan.indexPattern(), plan.indexMode(), plan.indexNameWithModes(), attributes, plan.query()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java index f66ed5c8e4ec1..d70153258871e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -84,7 +85,9 @@ && allowedForDocValues(fieldAttribute, ctx.searchStats(), agg, foundAttributes)) // We need to both mark the field to load differently, and change the spatial function to know to use it foundAttributes.add(fieldAttribute); changedAggregates = true; - orderedAggregates.add(as.replaceChild(af.withDocValues())); + orderedAggregates.add( + as.replaceChild(af.withFieldExtractPreference(MappedFieldType.FieldExtractPreference.DOC_VALUES)) + ); } else { orderedAggregates.add(aggExpr); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java index d284d2d61b0e7..3dd04e296f436 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialShapeBoundsExtraction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.lucene.spatial.GeometryDocValueWriter; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -49,9 +50,20 @@ public class SpatialShapeBoundsExtraction extends ParameterizedOptimizerRule { @Override protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { - var foundAttributes = new HashSet(); + Set foundAttributes = findSpatialShapeBoundsAttributes(aggregate, ctx); + if (foundAttributes.isEmpty()) { + return aggregate; + } + return aggregate.transformDown(PhysicalPlan.class, exec -> switch (exec.getClass().getSimpleName()) { + case "AggregateExec" -> transformAggregateExec((AggregateExec) exec, foundAttributes); + case "FieldExtractExec" -> transformFieldExtractExec((FieldExtractExec) exec, foundAttributes); + default -> exec; + }); + } - return aggregate.transformDown(UnaryExec.class, exec -> { + private static Set findSpatialShapeBoundsAttributes(AggregateExec aggregate, LocalPhysicalOptimizerContext ctx) { + var foundAttributes = new HashSet(); + aggregate.transformDown(UnaryExec.class, exec -> { if (exec instanceof AggregateExec agg) { List aggregateFunctions = agg.aggregates() .stream() @@ -84,15 +96,25 @@ protected PhysicalPlan rule(AggregateExec aggregate, LocalPhysicalOptimizerConte foundAttributes.removeAll(evalExec.references()); } else if (exec instanceof FilterExec filterExec) { foundAttributes.removeAll(filterExec.condition().references()); - } else if (exec instanceof FieldExtractExec fieldExtractExec) { - var boundsAttributes = new HashSet<>(foundAttributes); - boundsAttributes.retainAll(fieldExtractExec.attributesToExtract()); - if (boundsAttributes.isEmpty() == false) { - exec = fieldExtractExec.withBoundsAttributes(boundsAttributes); - } } return exec; }); + return foundAttributes; + } + + private static PhysicalPlan transformFieldExtractExec(FieldExtractExec fieldExtractExec, Set foundAttributes) { + var boundsAttributes = new HashSet<>(foundAttributes); + boundsAttributes.retainAll(fieldExtractExec.attributesToExtract()); + return fieldExtractExec.withBoundsAttributes(boundsAttributes); + } + + private static PhysicalPlan transformAggregateExec(AggregateExec agg, Set foundAttributes) { + return agg.transformExpressionsDown( + SpatialExtent.class, + spatialExtent -> foundAttributes.contains(spatialExtent.field()) + ? spatialExtent.withFieldExtractPreference(MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) + : spatialExtent + ); } private static boolean isShape(DataType dataType) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 1d050bd91e66c..b2a1cec46d67d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -16,8 +16,7 @@ null 'sort' 'stats' 'where' -null -null +'lookup' null null null @@ -120,6 +119,7 @@ null null null null +'join' 'USING' null null @@ -149,14 +149,13 @@ SHOW SORT STATS WHERE +JOIN_LOOKUP DEV_INLINESTATS DEV_LOOKUP DEV_METRICS -DEV_JOIN DEV_JOIN_FULL DEV_JOIN_LEFT DEV_JOIN_RIGHT -DEV_JOIN_LOOKUP UNKNOWN_CMD LINE_COMMENT MULTILINE_COMMENT @@ -253,6 +252,7 @@ LOOKUP_WS LOOKUP_FIELD_LINE_COMMENT LOOKUP_FIELD_MULTILINE_COMMENT LOOKUP_FIELD_WS +JOIN USING JOIN_LINE_COMMENT JOIN_MULTILINE_COMMENT @@ -281,14 +281,13 @@ SHOW SORT STATS WHERE +JOIN_LOOKUP DEV_INLINESTATS DEV_LOOKUP DEV_METRICS -DEV_JOIN DEV_JOIN_FULL DEV_JOIN_LEFT DEV_JOIN_RIGHT -DEV_JOIN_LOOKUP UNKNOWN_CMD LINE_COMMENT MULTILINE_COMMENT @@ -456,7 +455,7 @@ LOOKUP_FIELD_LINE_COMMENT LOOKUP_FIELD_MULTILINE_COMMENT LOOKUP_FIELD_WS JOIN_PIPE -JOIN_JOIN +JOIN JOIN_AS JOIN_ON USING @@ -507,4 +506,4 @@ METRICS_MODE CLOSING_METRICS_MODE atn: -[4, 0, 130, 1627, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 2, 198, 7, 198, 2, 199, 7, 199, 2, 200, 7, 200, 2, 201, 7, 201, 2, 202, 7, 202, 2, 203, 7, 203, 2, 204, 7, 204, 2, 205, 7, 205, 2, 206, 7, 206, 2, 207, 7, 207, 2, 208, 7, 208, 2, 209, 7, 209, 2, 210, 7, 210, 2, 211, 7, 211, 2, 212, 7, 212, 2, 213, 7, 213, 2, 214, 7, 214, 2, 215, 7, 215, 2, 216, 7, 216, 2, 217, 7, 217, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 4, 24, 664, 8, 24, 11, 24, 12, 24, 665, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 674, 8, 25, 10, 25, 12, 25, 677, 9, 25, 1, 25, 3, 25, 680, 8, 25, 1, 25, 3, 25, 683, 8, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 692, 8, 26, 10, 26, 12, 26, 695, 9, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 4, 27, 703, 8, 27, 11, 27, 12, 27, 704, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 724, 8, 33, 1, 33, 4, 33, 727, 8, 33, 11, 33, 12, 33, 728, 1, 34, 1, 34, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 3, 36, 738, 8, 36, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 3, 38, 745, 8, 38, 1, 39, 1, 39, 1, 39, 5, 39, 750, 8, 39, 10, 39, 12, 39, 753, 9, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 5, 39, 761, 8, 39, 10, 39, 12, 39, 764, 9, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 771, 8, 39, 1, 39, 3, 39, 774, 8, 39, 3, 39, 776, 8, 39, 1, 40, 4, 40, 779, 8, 40, 11, 40, 12, 40, 780, 1, 41, 4, 41, 784, 8, 41, 11, 41, 12, 41, 785, 1, 41, 1, 41, 5, 41, 790, 8, 41, 10, 41, 12, 41, 793, 9, 41, 1, 41, 1, 41, 4, 41, 797, 8, 41, 11, 41, 12, 41, 798, 1, 41, 4, 41, 802, 8, 41, 11, 41, 12, 41, 803, 1, 41, 1, 41, 5, 41, 808, 8, 41, 10, 41, 12, 41, 811, 9, 41, 3, 41, 813, 8, 41, 1, 41, 1, 41, 1, 41, 1, 41, 4, 41, 819, 8, 41, 11, 41, 12, 41, 820, 1, 41, 1, 41, 3, 41, 825, 8, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 3, 81, 957, 8, 81, 1, 81, 5, 81, 960, 8, 81, 10, 81, 12, 81, 963, 9, 81, 1, 81, 1, 81, 4, 81, 967, 8, 81, 11, 81, 12, 81, 968, 3, 81, 971, 8, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 5, 84, 985, 8, 84, 10, 84, 12, 84, 988, 9, 84, 1, 84, 1, 84, 3, 84, 992, 8, 84, 1, 84, 4, 84, 995, 8, 84, 11, 84, 12, 84, 996, 3, 84, 999, 8, 84, 1, 85, 1, 85, 4, 85, 1003, 8, 85, 11, 85, 12, 85, 1004, 1, 85, 1, 85, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 3, 102, 1082, 8, 102, 1, 103, 4, 103, 1085, 8, 103, 11, 103, 12, 103, 1086, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 3, 114, 1136, 8, 114, 1, 115, 1, 115, 3, 115, 1140, 8, 115, 1, 115, 5, 115, 1143, 8, 115, 10, 115, 12, 115, 1146, 9, 115, 1, 115, 1, 115, 3, 115, 1150, 8, 115, 1, 115, 4, 115, 1153, 8, 115, 11, 115, 12, 115, 1154, 3, 115, 1157, 8, 115, 1, 116, 1, 116, 4, 116, 1161, 8, 116, 11, 116, 12, 116, 1162, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 136, 4, 136, 1248, 8, 136, 11, 136, 12, 136, 1249, 1, 136, 1, 136, 3, 136, 1254, 8, 136, 1, 136, 4, 136, 1257, 8, 136, 11, 136, 12, 136, 1258, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 4, 169, 1404, 8, 169, 11, 169, 12, 169, 1405, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 198, 1, 198, 1, 198, 1, 198, 1, 199, 1, 199, 1, 199, 1, 199, 1, 200, 1, 200, 1, 200, 1, 200, 1, 201, 1, 201, 1, 201, 1, 201, 1, 202, 1, 202, 1, 202, 1, 202, 1, 203, 1, 203, 1, 203, 1, 203, 1, 203, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 206, 1, 206, 1, 206, 1, 206, 1, 207, 1, 207, 1, 207, 1, 207, 1, 208, 1, 208, 1, 208, 1, 208, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 210, 1, 210, 1, 210, 1, 210, 1, 210, 1, 210, 1, 211, 1, 211, 1, 211, 1, 211, 1, 212, 1, 212, 1, 212, 1, 212, 1, 213, 1, 213, 1, 213, 1, 213, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 217, 1, 217, 1, 217, 1, 217, 1, 217, 2, 693, 762, 0, 218, 16, 1, 18, 2, 20, 3, 22, 4, 24, 5, 26, 6, 28, 7, 30, 8, 32, 9, 34, 10, 36, 11, 38, 12, 40, 13, 42, 14, 44, 15, 46, 16, 48, 17, 50, 18, 52, 19, 54, 20, 56, 21, 58, 22, 60, 23, 62, 24, 64, 25, 66, 26, 68, 27, 70, 28, 72, 29, 74, 0, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 0, 88, 0, 90, 0, 92, 0, 94, 30, 96, 31, 98, 32, 100, 33, 102, 34, 104, 35, 106, 36, 108, 37, 110, 38, 112, 39, 114, 40, 116, 41, 118, 42, 120, 43, 122, 44, 124, 45, 126, 46, 128, 47, 130, 48, 132, 49, 134, 50, 136, 51, 138, 52, 140, 53, 142, 54, 144, 55, 146, 56, 148, 57, 150, 58, 152, 59, 154, 60, 156, 61, 158, 62, 160, 63, 162, 64, 164, 65, 166, 66, 168, 67, 170, 68, 172, 69, 174, 70, 176, 0, 178, 71, 180, 72, 182, 73, 184, 74, 186, 0, 188, 75, 190, 76, 192, 77, 194, 78, 196, 0, 198, 0, 200, 79, 202, 80, 204, 81, 206, 0, 208, 0, 210, 0, 212, 0, 214, 0, 216, 0, 218, 82, 220, 0, 222, 83, 224, 0, 226, 0, 228, 84, 230, 85, 232, 86, 234, 0, 236, 0, 238, 0, 240, 0, 242, 0, 244, 0, 246, 0, 248, 87, 250, 88, 252, 89, 254, 90, 256, 0, 258, 0, 260, 0, 262, 0, 264, 0, 266, 0, 268, 91, 270, 0, 272, 92, 274, 93, 276, 94, 278, 0, 280, 0, 282, 95, 284, 96, 286, 0, 288, 97, 290, 0, 292, 98, 294, 99, 296, 100, 298, 0, 300, 0, 302, 0, 304, 0, 306, 0, 308, 0, 310, 0, 312, 0, 314, 0, 316, 101, 318, 102, 320, 103, 322, 0, 324, 0, 326, 0, 328, 0, 330, 0, 332, 0, 334, 104, 336, 105, 338, 106, 340, 0, 342, 107, 344, 108, 346, 109, 348, 110, 350, 0, 352, 0, 354, 111, 356, 112, 358, 113, 360, 114, 362, 0, 364, 0, 366, 0, 368, 0, 370, 0, 372, 0, 374, 0, 376, 115, 378, 116, 380, 117, 382, 0, 384, 0, 386, 0, 388, 0, 390, 118, 392, 119, 394, 120, 396, 0, 398, 0, 400, 0, 402, 0, 404, 121, 406, 0, 408, 0, 410, 0, 412, 0, 414, 0, 416, 122, 418, 123, 420, 124, 422, 0, 424, 0, 426, 0, 428, 125, 430, 126, 432, 127, 434, 0, 436, 0, 438, 128, 440, 129, 442, 130, 444, 0, 446, 0, 448, 0, 450, 0, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 36, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 2, 0, 74, 74, 106, 106, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1654, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 66, 1, 0, 0, 0, 0, 68, 1, 0, 0, 0, 0, 70, 1, 0, 0, 0, 1, 72, 1, 0, 0, 0, 1, 94, 1, 0, 0, 0, 1, 96, 1, 0, 0, 0, 1, 98, 1, 0, 0, 0, 1, 100, 1, 0, 0, 0, 1, 102, 1, 0, 0, 0, 1, 104, 1, 0, 0, 0, 1, 106, 1, 0, 0, 0, 1, 108, 1, 0, 0, 0, 1, 110, 1, 0, 0, 0, 1, 112, 1, 0, 0, 0, 1, 114, 1, 0, 0, 0, 1, 116, 1, 0, 0, 0, 1, 118, 1, 0, 0, 0, 1, 120, 1, 0, 0, 0, 1, 122, 1, 0, 0, 0, 1, 124, 1, 0, 0, 0, 1, 126, 1, 0, 0, 0, 1, 128, 1, 0, 0, 0, 1, 130, 1, 0, 0, 0, 1, 132, 1, 0, 0, 0, 1, 134, 1, 0, 0, 0, 1, 136, 1, 0, 0, 0, 1, 138, 1, 0, 0, 0, 1, 140, 1, 0, 0, 0, 1, 142, 1, 0, 0, 0, 1, 144, 1, 0, 0, 0, 1, 146, 1, 0, 0, 0, 1, 148, 1, 0, 0, 0, 1, 150, 1, 0, 0, 0, 1, 152, 1, 0, 0, 0, 1, 154, 1, 0, 0, 0, 1, 156, 1, 0, 0, 0, 1, 158, 1, 0, 0, 0, 1, 160, 1, 0, 0, 0, 1, 162, 1, 0, 0, 0, 1, 164, 1, 0, 0, 0, 1, 166, 1, 0, 0, 0, 1, 168, 1, 0, 0, 0, 1, 170, 1, 0, 0, 0, 1, 172, 1, 0, 0, 0, 1, 174, 1, 0, 0, 0, 1, 176, 1, 0, 0, 0, 1, 178, 1, 0, 0, 0, 1, 180, 1, 0, 0, 0, 1, 182, 1, 0, 0, 0, 1, 184, 1, 0, 0, 0, 1, 188, 1, 0, 0, 0, 1, 190, 1, 0, 0, 0, 1, 192, 1, 0, 0, 0, 1, 194, 1, 0, 0, 0, 2, 196, 1, 0, 0, 0, 2, 198, 1, 0, 0, 0, 2, 200, 1, 0, 0, 0, 2, 202, 1, 0, 0, 0, 2, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 208, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 3, 214, 1, 0, 0, 0, 3, 216, 1, 0, 0, 0, 3, 218, 1, 0, 0, 0, 3, 222, 1, 0, 0, 0, 3, 224, 1, 0, 0, 0, 3, 226, 1, 0, 0, 0, 3, 228, 1, 0, 0, 0, 3, 230, 1, 0, 0, 0, 3, 232, 1, 0, 0, 0, 4, 234, 1, 0, 0, 0, 4, 236, 1, 0, 0, 0, 4, 238, 1, 0, 0, 0, 4, 240, 1, 0, 0, 0, 4, 242, 1, 0, 0, 0, 4, 248, 1, 0, 0, 0, 4, 250, 1, 0, 0, 0, 4, 252, 1, 0, 0, 0, 4, 254, 1, 0, 0, 0, 5, 256, 1, 0, 0, 0, 5, 258, 1, 0, 0, 0, 5, 260, 1, 0, 0, 0, 5, 262, 1, 0, 0, 0, 5, 264, 1, 0, 0, 0, 5, 266, 1, 0, 0, 0, 5, 268, 1, 0, 0, 0, 5, 270, 1, 0, 0, 0, 5, 272, 1, 0, 0, 0, 5, 274, 1, 0, 0, 0, 5, 276, 1, 0, 0, 0, 6, 278, 1, 0, 0, 0, 6, 280, 1, 0, 0, 0, 6, 282, 1, 0, 0, 0, 6, 284, 1, 0, 0, 0, 6, 288, 1, 0, 0, 0, 6, 290, 1, 0, 0, 0, 6, 292, 1, 0, 0, 0, 6, 294, 1, 0, 0, 0, 6, 296, 1, 0, 0, 0, 7, 298, 1, 0, 0, 0, 7, 300, 1, 0, 0, 0, 7, 302, 1, 0, 0, 0, 7, 304, 1, 0, 0, 0, 7, 306, 1, 0, 0, 0, 7, 308, 1, 0, 0, 0, 7, 310, 1, 0, 0, 0, 7, 312, 1, 0, 0, 0, 7, 314, 1, 0, 0, 0, 7, 316, 1, 0, 0, 0, 7, 318, 1, 0, 0, 0, 7, 320, 1, 0, 0, 0, 8, 322, 1, 0, 0, 0, 8, 324, 1, 0, 0, 0, 8, 326, 1, 0, 0, 0, 8, 328, 1, 0, 0, 0, 8, 330, 1, 0, 0, 0, 8, 332, 1, 0, 0, 0, 8, 334, 1, 0, 0, 0, 8, 336, 1, 0, 0, 0, 8, 338, 1, 0, 0, 0, 9, 340, 1, 0, 0, 0, 9, 342, 1, 0, 0, 0, 9, 344, 1, 0, 0, 0, 9, 346, 1, 0, 0, 0, 9, 348, 1, 0, 0, 0, 10, 350, 1, 0, 0, 0, 10, 352, 1, 0, 0, 0, 10, 354, 1, 0, 0, 0, 10, 356, 1, 0, 0, 0, 10, 358, 1, 0, 0, 0, 10, 360, 1, 0, 0, 0, 11, 362, 1, 0, 0, 0, 11, 364, 1, 0, 0, 0, 11, 366, 1, 0, 0, 0, 11, 368, 1, 0, 0, 0, 11, 370, 1, 0, 0, 0, 11, 372, 1, 0, 0, 0, 11, 374, 1, 0, 0, 0, 11, 376, 1, 0, 0, 0, 11, 378, 1, 0, 0, 0, 11, 380, 1, 0, 0, 0, 12, 382, 1, 0, 0, 0, 12, 384, 1, 0, 0, 0, 12, 386, 1, 0, 0, 0, 12, 388, 1, 0, 0, 0, 12, 390, 1, 0, 0, 0, 12, 392, 1, 0, 0, 0, 12, 394, 1, 0, 0, 0, 13, 396, 1, 0, 0, 0, 13, 398, 1, 0, 0, 0, 13, 400, 1, 0, 0, 0, 13, 402, 1, 0, 0, 0, 13, 404, 1, 0, 0, 0, 13, 406, 1, 0, 0, 0, 13, 408, 1, 0, 0, 0, 13, 410, 1, 0, 0, 0, 13, 412, 1, 0, 0, 0, 13, 414, 1, 0, 0, 0, 13, 416, 1, 0, 0, 0, 13, 418, 1, 0, 0, 0, 13, 420, 1, 0, 0, 0, 14, 422, 1, 0, 0, 0, 14, 424, 1, 0, 0, 0, 14, 426, 1, 0, 0, 0, 14, 428, 1, 0, 0, 0, 14, 430, 1, 0, 0, 0, 14, 432, 1, 0, 0, 0, 15, 434, 1, 0, 0, 0, 15, 436, 1, 0, 0, 0, 15, 438, 1, 0, 0, 0, 15, 440, 1, 0, 0, 0, 15, 442, 1, 0, 0, 0, 15, 444, 1, 0, 0, 0, 15, 446, 1, 0, 0, 0, 15, 448, 1, 0, 0, 0, 15, 450, 1, 0, 0, 0, 16, 452, 1, 0, 0, 0, 18, 462, 1, 0, 0, 0, 20, 469, 1, 0, 0, 0, 22, 478, 1, 0, 0, 0, 24, 485, 1, 0, 0, 0, 26, 495, 1, 0, 0, 0, 28, 502, 1, 0, 0, 0, 30, 509, 1, 0, 0, 0, 32, 516, 1, 0, 0, 0, 34, 524, 1, 0, 0, 0, 36, 536, 1, 0, 0, 0, 38, 545, 1, 0, 0, 0, 40, 551, 1, 0, 0, 0, 42, 558, 1, 0, 0, 0, 44, 565, 1, 0, 0, 0, 46, 573, 1, 0, 0, 0, 48, 581, 1, 0, 0, 0, 50, 596, 1, 0, 0, 0, 52, 608, 1, 0, 0, 0, 54, 619, 1, 0, 0, 0, 56, 627, 1, 0, 0, 0, 58, 635, 1, 0, 0, 0, 60, 643, 1, 0, 0, 0, 62, 652, 1, 0, 0, 0, 64, 663, 1, 0, 0, 0, 66, 669, 1, 0, 0, 0, 68, 686, 1, 0, 0, 0, 70, 702, 1, 0, 0, 0, 72, 708, 1, 0, 0, 0, 74, 712, 1, 0, 0, 0, 76, 714, 1, 0, 0, 0, 78, 716, 1, 0, 0, 0, 80, 719, 1, 0, 0, 0, 82, 721, 1, 0, 0, 0, 84, 730, 1, 0, 0, 0, 86, 732, 1, 0, 0, 0, 88, 737, 1, 0, 0, 0, 90, 739, 1, 0, 0, 0, 92, 744, 1, 0, 0, 0, 94, 775, 1, 0, 0, 0, 96, 778, 1, 0, 0, 0, 98, 824, 1, 0, 0, 0, 100, 826, 1, 0, 0, 0, 102, 829, 1, 0, 0, 0, 104, 833, 1, 0, 0, 0, 106, 837, 1, 0, 0, 0, 108, 839, 1, 0, 0, 0, 110, 842, 1, 0, 0, 0, 112, 844, 1, 0, 0, 0, 114, 846, 1, 0, 0, 0, 116, 851, 1, 0, 0, 0, 118, 853, 1, 0, 0, 0, 120, 859, 1, 0, 0, 0, 122, 865, 1, 0, 0, 0, 124, 868, 1, 0, 0, 0, 126, 871, 1, 0, 0, 0, 128, 876, 1, 0, 0, 0, 130, 881, 1, 0, 0, 0, 132, 883, 1, 0, 0, 0, 134, 887, 1, 0, 0, 0, 136, 892, 1, 0, 0, 0, 138, 898, 1, 0, 0, 0, 140, 901, 1, 0, 0, 0, 142, 903, 1, 0, 0, 0, 144, 909, 1, 0, 0, 0, 146, 911, 1, 0, 0, 0, 148, 916, 1, 0, 0, 0, 150, 919, 1, 0, 0, 0, 152, 922, 1, 0, 0, 0, 154, 925, 1, 0, 0, 0, 156, 927, 1, 0, 0, 0, 158, 930, 1, 0, 0, 0, 160, 932, 1, 0, 0, 0, 162, 935, 1, 0, 0, 0, 164, 937, 1, 0, 0, 0, 166, 939, 1, 0, 0, 0, 168, 941, 1, 0, 0, 0, 170, 943, 1, 0, 0, 0, 172, 945, 1, 0, 0, 0, 174, 947, 1, 0, 0, 0, 176, 949, 1, 0, 0, 0, 178, 970, 1, 0, 0, 0, 180, 972, 1, 0, 0, 0, 182, 977, 1, 0, 0, 0, 184, 998, 1, 0, 0, 0, 186, 1000, 1, 0, 0, 0, 188, 1008, 1, 0, 0, 0, 190, 1010, 1, 0, 0, 0, 192, 1014, 1, 0, 0, 0, 194, 1018, 1, 0, 0, 0, 196, 1022, 1, 0, 0, 0, 198, 1027, 1, 0, 0, 0, 200, 1032, 1, 0, 0, 0, 202, 1036, 1, 0, 0, 0, 204, 1040, 1, 0, 0, 0, 206, 1044, 1, 0, 0, 0, 208, 1049, 1, 0, 0, 0, 210, 1053, 1, 0, 0, 0, 212, 1057, 1, 0, 0, 0, 214, 1061, 1, 0, 0, 0, 216, 1065, 1, 0, 0, 0, 218, 1069, 1, 0, 0, 0, 220, 1081, 1, 0, 0, 0, 222, 1084, 1, 0, 0, 0, 224, 1088, 1, 0, 0, 0, 226, 1092, 1, 0, 0, 0, 228, 1096, 1, 0, 0, 0, 230, 1100, 1, 0, 0, 0, 232, 1104, 1, 0, 0, 0, 234, 1108, 1, 0, 0, 0, 236, 1113, 1, 0, 0, 0, 238, 1117, 1, 0, 0, 0, 240, 1121, 1, 0, 0, 0, 242, 1126, 1, 0, 0, 0, 244, 1135, 1, 0, 0, 0, 246, 1156, 1, 0, 0, 0, 248, 1160, 1, 0, 0, 0, 250, 1164, 1, 0, 0, 0, 252, 1168, 1, 0, 0, 0, 254, 1172, 1, 0, 0, 0, 256, 1176, 1, 0, 0, 0, 258, 1181, 1, 0, 0, 0, 260, 1185, 1, 0, 0, 0, 262, 1189, 1, 0, 0, 0, 264, 1193, 1, 0, 0, 0, 266, 1198, 1, 0, 0, 0, 268, 1203, 1, 0, 0, 0, 270, 1206, 1, 0, 0, 0, 272, 1210, 1, 0, 0, 0, 274, 1214, 1, 0, 0, 0, 276, 1218, 1, 0, 0, 0, 278, 1222, 1, 0, 0, 0, 280, 1227, 1, 0, 0, 0, 282, 1232, 1, 0, 0, 0, 284, 1237, 1, 0, 0, 0, 286, 1244, 1, 0, 0, 0, 288, 1253, 1, 0, 0, 0, 290, 1260, 1, 0, 0, 0, 292, 1264, 1, 0, 0, 0, 294, 1268, 1, 0, 0, 0, 296, 1272, 1, 0, 0, 0, 298, 1276, 1, 0, 0, 0, 300, 1282, 1, 0, 0, 0, 302, 1286, 1, 0, 0, 0, 304, 1290, 1, 0, 0, 0, 306, 1294, 1, 0, 0, 0, 308, 1298, 1, 0, 0, 0, 310, 1302, 1, 0, 0, 0, 312, 1306, 1, 0, 0, 0, 314, 1311, 1, 0, 0, 0, 316, 1316, 1, 0, 0, 0, 318, 1320, 1, 0, 0, 0, 320, 1324, 1, 0, 0, 0, 322, 1328, 1, 0, 0, 0, 324, 1333, 1, 0, 0, 0, 326, 1337, 1, 0, 0, 0, 328, 1342, 1, 0, 0, 0, 330, 1347, 1, 0, 0, 0, 332, 1351, 1, 0, 0, 0, 334, 1355, 1, 0, 0, 0, 336, 1359, 1, 0, 0, 0, 338, 1363, 1, 0, 0, 0, 340, 1367, 1, 0, 0, 0, 342, 1372, 1, 0, 0, 0, 344, 1377, 1, 0, 0, 0, 346, 1381, 1, 0, 0, 0, 348, 1385, 1, 0, 0, 0, 350, 1389, 1, 0, 0, 0, 352, 1394, 1, 0, 0, 0, 354, 1403, 1, 0, 0, 0, 356, 1407, 1, 0, 0, 0, 358, 1411, 1, 0, 0, 0, 360, 1415, 1, 0, 0, 0, 362, 1419, 1, 0, 0, 0, 364, 1424, 1, 0, 0, 0, 366, 1428, 1, 0, 0, 0, 368, 1432, 1, 0, 0, 0, 370, 1436, 1, 0, 0, 0, 372, 1441, 1, 0, 0, 0, 374, 1445, 1, 0, 0, 0, 376, 1449, 1, 0, 0, 0, 378, 1453, 1, 0, 0, 0, 380, 1457, 1, 0, 0, 0, 382, 1461, 1, 0, 0, 0, 384, 1467, 1, 0, 0, 0, 386, 1471, 1, 0, 0, 0, 388, 1475, 1, 0, 0, 0, 390, 1479, 1, 0, 0, 0, 392, 1483, 1, 0, 0, 0, 394, 1487, 1, 0, 0, 0, 396, 1491, 1, 0, 0, 0, 398, 1496, 1, 0, 0, 0, 400, 1500, 1, 0, 0, 0, 402, 1504, 1, 0, 0, 0, 404, 1510, 1, 0, 0, 0, 406, 1519, 1, 0, 0, 0, 408, 1523, 1, 0, 0, 0, 410, 1527, 1, 0, 0, 0, 412, 1531, 1, 0, 0, 0, 414, 1535, 1, 0, 0, 0, 416, 1539, 1, 0, 0, 0, 418, 1543, 1, 0, 0, 0, 420, 1547, 1, 0, 0, 0, 422, 1551, 1, 0, 0, 0, 424, 1556, 1, 0, 0, 0, 426, 1562, 1, 0, 0, 0, 428, 1568, 1, 0, 0, 0, 430, 1572, 1, 0, 0, 0, 432, 1576, 1, 0, 0, 0, 434, 1580, 1, 0, 0, 0, 436, 1586, 1, 0, 0, 0, 438, 1592, 1, 0, 0, 0, 440, 1596, 1, 0, 0, 0, 442, 1600, 1, 0, 0, 0, 444, 1604, 1, 0, 0, 0, 446, 1610, 1, 0, 0, 0, 448, 1616, 1, 0, 0, 0, 450, 1622, 1, 0, 0, 0, 452, 453, 7, 0, 0, 0, 453, 454, 7, 1, 0, 0, 454, 455, 7, 2, 0, 0, 455, 456, 7, 2, 0, 0, 456, 457, 7, 3, 0, 0, 457, 458, 7, 4, 0, 0, 458, 459, 7, 5, 0, 0, 459, 460, 1, 0, 0, 0, 460, 461, 6, 0, 0, 0, 461, 17, 1, 0, 0, 0, 462, 463, 7, 0, 0, 0, 463, 464, 7, 6, 0, 0, 464, 465, 7, 7, 0, 0, 465, 466, 7, 8, 0, 0, 466, 467, 1, 0, 0, 0, 467, 468, 6, 1, 1, 0, 468, 19, 1, 0, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 9, 0, 0, 471, 472, 7, 6, 0, 0, 472, 473, 7, 1, 0, 0, 473, 474, 7, 4, 0, 0, 474, 475, 7, 10, 0, 0, 475, 476, 1, 0, 0, 0, 476, 477, 6, 2, 2, 0, 477, 21, 1, 0, 0, 0, 478, 479, 7, 3, 0, 0, 479, 480, 7, 11, 0, 0, 480, 481, 7, 12, 0, 0, 481, 482, 7, 13, 0, 0, 482, 483, 1, 0, 0, 0, 483, 484, 6, 3, 0, 0, 484, 23, 1, 0, 0, 0, 485, 486, 7, 3, 0, 0, 486, 487, 7, 14, 0, 0, 487, 488, 7, 8, 0, 0, 488, 489, 7, 13, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 1, 0, 0, 491, 492, 7, 9, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 4, 3, 0, 494, 25, 1, 0, 0, 0, 495, 496, 7, 15, 0, 0, 496, 497, 7, 6, 0, 0, 497, 498, 7, 7, 0, 0, 498, 499, 7, 16, 0, 0, 499, 500, 1, 0, 0, 0, 500, 501, 6, 5, 4, 0, 501, 27, 1, 0, 0, 0, 502, 503, 7, 17, 0, 0, 503, 504, 7, 6, 0, 0, 504, 505, 7, 7, 0, 0, 505, 506, 7, 18, 0, 0, 506, 507, 1, 0, 0, 0, 507, 508, 6, 6, 0, 0, 508, 29, 1, 0, 0, 0, 509, 510, 7, 18, 0, 0, 510, 511, 7, 3, 0, 0, 511, 512, 7, 3, 0, 0, 512, 513, 7, 8, 0, 0, 513, 514, 1, 0, 0, 0, 514, 515, 6, 7, 1, 0, 515, 31, 1, 0, 0, 0, 516, 517, 7, 13, 0, 0, 517, 518, 7, 1, 0, 0, 518, 519, 7, 16, 0, 0, 519, 520, 7, 1, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 8, 0, 0, 523, 33, 1, 0, 0, 0, 524, 525, 7, 16, 0, 0, 525, 526, 7, 11, 0, 0, 526, 527, 5, 95, 0, 0, 527, 528, 7, 3, 0, 0, 528, 529, 7, 14, 0, 0, 529, 530, 7, 8, 0, 0, 530, 531, 7, 12, 0, 0, 531, 532, 7, 9, 0, 0, 532, 533, 7, 0, 0, 0, 533, 534, 1, 0, 0, 0, 534, 535, 6, 9, 5, 0, 535, 35, 1, 0, 0, 0, 536, 537, 7, 6, 0, 0, 537, 538, 7, 3, 0, 0, 538, 539, 7, 9, 0, 0, 539, 540, 7, 12, 0, 0, 540, 541, 7, 16, 0, 0, 541, 542, 7, 3, 0, 0, 542, 543, 1, 0, 0, 0, 543, 544, 6, 10, 6, 0, 544, 37, 1, 0, 0, 0, 545, 546, 7, 6, 0, 0, 546, 547, 7, 7, 0, 0, 547, 548, 7, 19, 0, 0, 548, 549, 1, 0, 0, 0, 549, 550, 6, 11, 0, 0, 550, 39, 1, 0, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 7, 10, 0, 0, 553, 554, 7, 7, 0, 0, 554, 555, 7, 19, 0, 0, 555, 556, 1, 0, 0, 0, 556, 557, 6, 12, 7, 0, 557, 41, 1, 0, 0, 0, 558, 559, 7, 2, 0, 0, 559, 560, 7, 7, 0, 0, 560, 561, 7, 6, 0, 0, 561, 562, 7, 5, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 13, 0, 0, 564, 43, 1, 0, 0, 0, 565, 566, 7, 2, 0, 0, 566, 567, 7, 5, 0, 0, 567, 568, 7, 12, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 2, 0, 0, 570, 571, 1, 0, 0, 0, 571, 572, 6, 14, 0, 0, 572, 45, 1, 0, 0, 0, 573, 574, 7, 19, 0, 0, 574, 575, 7, 10, 0, 0, 575, 576, 7, 3, 0, 0, 576, 577, 7, 6, 0, 0, 577, 578, 7, 3, 0, 0, 578, 579, 1, 0, 0, 0, 579, 580, 6, 15, 0, 0, 580, 47, 1, 0, 0, 0, 581, 582, 4, 16, 0, 0, 582, 583, 7, 1, 0, 0, 583, 584, 7, 9, 0, 0, 584, 585, 7, 13, 0, 0, 585, 586, 7, 1, 0, 0, 586, 587, 7, 9, 0, 0, 587, 588, 7, 3, 0, 0, 588, 589, 7, 2, 0, 0, 589, 590, 7, 5, 0, 0, 590, 591, 7, 12, 0, 0, 591, 592, 7, 5, 0, 0, 592, 593, 7, 2, 0, 0, 593, 594, 1, 0, 0, 0, 594, 595, 6, 16, 0, 0, 595, 49, 1, 0, 0, 0, 596, 597, 4, 17, 1, 0, 597, 598, 7, 13, 0, 0, 598, 599, 7, 7, 0, 0, 599, 600, 7, 7, 0, 0, 600, 601, 7, 18, 0, 0, 601, 602, 7, 20, 0, 0, 602, 603, 7, 8, 0, 0, 603, 604, 5, 95, 0, 0, 604, 605, 5, 128020, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 6, 17, 8, 0, 607, 51, 1, 0, 0, 0, 608, 609, 4, 18, 2, 0, 609, 610, 7, 16, 0, 0, 610, 611, 7, 3, 0, 0, 611, 612, 7, 5, 0, 0, 612, 613, 7, 6, 0, 0, 613, 614, 7, 1, 0, 0, 614, 615, 7, 4, 0, 0, 615, 616, 7, 2, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 6, 18, 9, 0, 618, 53, 1, 0, 0, 0, 619, 620, 4, 19, 3, 0, 620, 621, 7, 21, 0, 0, 621, 622, 7, 7, 0, 0, 622, 623, 7, 1, 0, 0, 623, 624, 7, 9, 0, 0, 624, 625, 1, 0, 0, 0, 625, 626, 6, 19, 10, 0, 626, 55, 1, 0, 0, 0, 627, 628, 4, 20, 4, 0, 628, 629, 7, 15, 0, 0, 629, 630, 7, 20, 0, 0, 630, 631, 7, 13, 0, 0, 631, 632, 7, 13, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 6, 20, 10, 0, 634, 57, 1, 0, 0, 0, 635, 636, 4, 21, 5, 0, 636, 637, 7, 13, 0, 0, 637, 638, 7, 3, 0, 0, 638, 639, 7, 15, 0, 0, 639, 640, 7, 5, 0, 0, 640, 641, 1, 0, 0, 0, 641, 642, 6, 21, 10, 0, 642, 59, 1, 0, 0, 0, 643, 644, 4, 22, 6, 0, 644, 645, 7, 6, 0, 0, 645, 646, 7, 1, 0, 0, 646, 647, 7, 17, 0, 0, 647, 648, 7, 10, 0, 0, 648, 649, 7, 5, 0, 0, 649, 650, 1, 0, 0, 0, 650, 651, 6, 22, 10, 0, 651, 61, 1, 0, 0, 0, 652, 653, 4, 23, 7, 0, 653, 654, 7, 13, 0, 0, 654, 655, 7, 7, 0, 0, 655, 656, 7, 7, 0, 0, 656, 657, 7, 18, 0, 0, 657, 658, 7, 20, 0, 0, 658, 659, 7, 8, 0, 0, 659, 660, 1, 0, 0, 0, 660, 661, 6, 23, 10, 0, 661, 63, 1, 0, 0, 0, 662, 664, 8, 22, 0, 0, 663, 662, 1, 0, 0, 0, 664, 665, 1, 0, 0, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 667, 1, 0, 0, 0, 667, 668, 6, 24, 0, 0, 668, 65, 1, 0, 0, 0, 669, 670, 5, 47, 0, 0, 670, 671, 5, 47, 0, 0, 671, 675, 1, 0, 0, 0, 672, 674, 8, 23, 0, 0, 673, 672, 1, 0, 0, 0, 674, 677, 1, 0, 0, 0, 675, 673, 1, 0, 0, 0, 675, 676, 1, 0, 0, 0, 676, 679, 1, 0, 0, 0, 677, 675, 1, 0, 0, 0, 678, 680, 5, 13, 0, 0, 679, 678, 1, 0, 0, 0, 679, 680, 1, 0, 0, 0, 680, 682, 1, 0, 0, 0, 681, 683, 5, 10, 0, 0, 682, 681, 1, 0, 0, 0, 682, 683, 1, 0, 0, 0, 683, 684, 1, 0, 0, 0, 684, 685, 6, 25, 11, 0, 685, 67, 1, 0, 0, 0, 686, 687, 5, 47, 0, 0, 687, 688, 5, 42, 0, 0, 688, 693, 1, 0, 0, 0, 689, 692, 3, 68, 26, 0, 690, 692, 9, 0, 0, 0, 691, 689, 1, 0, 0, 0, 691, 690, 1, 0, 0, 0, 692, 695, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 693, 691, 1, 0, 0, 0, 694, 696, 1, 0, 0, 0, 695, 693, 1, 0, 0, 0, 696, 697, 5, 42, 0, 0, 697, 698, 5, 47, 0, 0, 698, 699, 1, 0, 0, 0, 699, 700, 6, 26, 11, 0, 700, 69, 1, 0, 0, 0, 701, 703, 7, 24, 0, 0, 702, 701, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 702, 1, 0, 0, 0, 704, 705, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 706, 707, 6, 27, 11, 0, 707, 71, 1, 0, 0, 0, 708, 709, 5, 124, 0, 0, 709, 710, 1, 0, 0, 0, 710, 711, 6, 28, 12, 0, 711, 73, 1, 0, 0, 0, 712, 713, 7, 25, 0, 0, 713, 75, 1, 0, 0, 0, 714, 715, 7, 26, 0, 0, 715, 77, 1, 0, 0, 0, 716, 717, 5, 92, 0, 0, 717, 718, 7, 27, 0, 0, 718, 79, 1, 0, 0, 0, 719, 720, 8, 28, 0, 0, 720, 81, 1, 0, 0, 0, 721, 723, 7, 3, 0, 0, 722, 724, 7, 29, 0, 0, 723, 722, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 726, 1, 0, 0, 0, 725, 727, 3, 74, 29, 0, 726, 725, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 726, 1, 0, 0, 0, 728, 729, 1, 0, 0, 0, 729, 83, 1, 0, 0, 0, 730, 731, 5, 64, 0, 0, 731, 85, 1, 0, 0, 0, 732, 733, 5, 96, 0, 0, 733, 87, 1, 0, 0, 0, 734, 738, 8, 30, 0, 0, 735, 736, 5, 96, 0, 0, 736, 738, 5, 96, 0, 0, 737, 734, 1, 0, 0, 0, 737, 735, 1, 0, 0, 0, 738, 89, 1, 0, 0, 0, 739, 740, 5, 95, 0, 0, 740, 91, 1, 0, 0, 0, 741, 745, 3, 76, 30, 0, 742, 745, 3, 74, 29, 0, 743, 745, 3, 90, 37, 0, 744, 741, 1, 0, 0, 0, 744, 742, 1, 0, 0, 0, 744, 743, 1, 0, 0, 0, 745, 93, 1, 0, 0, 0, 746, 751, 5, 34, 0, 0, 747, 750, 3, 78, 31, 0, 748, 750, 3, 80, 32, 0, 749, 747, 1, 0, 0, 0, 749, 748, 1, 0, 0, 0, 750, 753, 1, 0, 0, 0, 751, 749, 1, 0, 0, 0, 751, 752, 1, 0, 0, 0, 752, 754, 1, 0, 0, 0, 753, 751, 1, 0, 0, 0, 754, 776, 5, 34, 0, 0, 755, 756, 5, 34, 0, 0, 756, 757, 5, 34, 0, 0, 757, 758, 5, 34, 0, 0, 758, 762, 1, 0, 0, 0, 759, 761, 8, 23, 0, 0, 760, 759, 1, 0, 0, 0, 761, 764, 1, 0, 0, 0, 762, 763, 1, 0, 0, 0, 762, 760, 1, 0, 0, 0, 763, 765, 1, 0, 0, 0, 764, 762, 1, 0, 0, 0, 765, 766, 5, 34, 0, 0, 766, 767, 5, 34, 0, 0, 767, 768, 5, 34, 0, 0, 768, 770, 1, 0, 0, 0, 769, 771, 5, 34, 0, 0, 770, 769, 1, 0, 0, 0, 770, 771, 1, 0, 0, 0, 771, 773, 1, 0, 0, 0, 772, 774, 5, 34, 0, 0, 773, 772, 1, 0, 0, 0, 773, 774, 1, 0, 0, 0, 774, 776, 1, 0, 0, 0, 775, 746, 1, 0, 0, 0, 775, 755, 1, 0, 0, 0, 776, 95, 1, 0, 0, 0, 777, 779, 3, 74, 29, 0, 778, 777, 1, 0, 0, 0, 779, 780, 1, 0, 0, 0, 780, 778, 1, 0, 0, 0, 780, 781, 1, 0, 0, 0, 781, 97, 1, 0, 0, 0, 782, 784, 3, 74, 29, 0, 783, 782, 1, 0, 0, 0, 784, 785, 1, 0, 0, 0, 785, 783, 1, 0, 0, 0, 785, 786, 1, 0, 0, 0, 786, 787, 1, 0, 0, 0, 787, 791, 3, 116, 50, 0, 788, 790, 3, 74, 29, 0, 789, 788, 1, 0, 0, 0, 790, 793, 1, 0, 0, 0, 791, 789, 1, 0, 0, 0, 791, 792, 1, 0, 0, 0, 792, 825, 1, 0, 0, 0, 793, 791, 1, 0, 0, 0, 794, 796, 3, 116, 50, 0, 795, 797, 3, 74, 29, 0, 796, 795, 1, 0, 0, 0, 797, 798, 1, 0, 0, 0, 798, 796, 1, 0, 0, 0, 798, 799, 1, 0, 0, 0, 799, 825, 1, 0, 0, 0, 800, 802, 3, 74, 29, 0, 801, 800, 1, 0, 0, 0, 802, 803, 1, 0, 0, 0, 803, 801, 1, 0, 0, 0, 803, 804, 1, 0, 0, 0, 804, 812, 1, 0, 0, 0, 805, 809, 3, 116, 50, 0, 806, 808, 3, 74, 29, 0, 807, 806, 1, 0, 0, 0, 808, 811, 1, 0, 0, 0, 809, 807, 1, 0, 0, 0, 809, 810, 1, 0, 0, 0, 810, 813, 1, 0, 0, 0, 811, 809, 1, 0, 0, 0, 812, 805, 1, 0, 0, 0, 812, 813, 1, 0, 0, 0, 813, 814, 1, 0, 0, 0, 814, 815, 3, 82, 33, 0, 815, 825, 1, 0, 0, 0, 816, 818, 3, 116, 50, 0, 817, 819, 3, 74, 29, 0, 818, 817, 1, 0, 0, 0, 819, 820, 1, 0, 0, 0, 820, 818, 1, 0, 0, 0, 820, 821, 1, 0, 0, 0, 821, 822, 1, 0, 0, 0, 822, 823, 3, 82, 33, 0, 823, 825, 1, 0, 0, 0, 824, 783, 1, 0, 0, 0, 824, 794, 1, 0, 0, 0, 824, 801, 1, 0, 0, 0, 824, 816, 1, 0, 0, 0, 825, 99, 1, 0, 0, 0, 826, 827, 7, 31, 0, 0, 827, 828, 7, 32, 0, 0, 828, 101, 1, 0, 0, 0, 829, 830, 7, 12, 0, 0, 830, 831, 7, 9, 0, 0, 831, 832, 7, 0, 0, 0, 832, 103, 1, 0, 0, 0, 833, 834, 7, 12, 0, 0, 834, 835, 7, 2, 0, 0, 835, 836, 7, 4, 0, 0, 836, 105, 1, 0, 0, 0, 837, 838, 5, 61, 0, 0, 838, 107, 1, 0, 0, 0, 839, 840, 5, 58, 0, 0, 840, 841, 5, 58, 0, 0, 841, 109, 1, 0, 0, 0, 842, 843, 5, 58, 0, 0, 843, 111, 1, 0, 0, 0, 844, 845, 5, 44, 0, 0, 845, 113, 1, 0, 0, 0, 846, 847, 7, 0, 0, 0, 847, 848, 7, 3, 0, 0, 848, 849, 7, 2, 0, 0, 849, 850, 7, 4, 0, 0, 850, 115, 1, 0, 0, 0, 851, 852, 5, 46, 0, 0, 852, 117, 1, 0, 0, 0, 853, 854, 7, 15, 0, 0, 854, 855, 7, 12, 0, 0, 855, 856, 7, 13, 0, 0, 856, 857, 7, 2, 0, 0, 857, 858, 7, 3, 0, 0, 858, 119, 1, 0, 0, 0, 859, 860, 7, 15, 0, 0, 860, 861, 7, 1, 0, 0, 861, 862, 7, 6, 0, 0, 862, 863, 7, 2, 0, 0, 863, 864, 7, 5, 0, 0, 864, 121, 1, 0, 0, 0, 865, 866, 7, 1, 0, 0, 866, 867, 7, 9, 0, 0, 867, 123, 1, 0, 0, 0, 868, 869, 7, 1, 0, 0, 869, 870, 7, 2, 0, 0, 870, 125, 1, 0, 0, 0, 871, 872, 7, 13, 0, 0, 872, 873, 7, 12, 0, 0, 873, 874, 7, 2, 0, 0, 874, 875, 7, 5, 0, 0, 875, 127, 1, 0, 0, 0, 876, 877, 7, 13, 0, 0, 877, 878, 7, 1, 0, 0, 878, 879, 7, 18, 0, 0, 879, 880, 7, 3, 0, 0, 880, 129, 1, 0, 0, 0, 881, 882, 5, 40, 0, 0, 882, 131, 1, 0, 0, 0, 883, 884, 7, 9, 0, 0, 884, 885, 7, 7, 0, 0, 885, 886, 7, 5, 0, 0, 886, 133, 1, 0, 0, 0, 887, 888, 7, 9, 0, 0, 888, 889, 7, 20, 0, 0, 889, 890, 7, 13, 0, 0, 890, 891, 7, 13, 0, 0, 891, 135, 1, 0, 0, 0, 892, 893, 7, 9, 0, 0, 893, 894, 7, 20, 0, 0, 894, 895, 7, 13, 0, 0, 895, 896, 7, 13, 0, 0, 896, 897, 7, 2, 0, 0, 897, 137, 1, 0, 0, 0, 898, 899, 7, 7, 0, 0, 899, 900, 7, 6, 0, 0, 900, 139, 1, 0, 0, 0, 901, 902, 5, 63, 0, 0, 902, 141, 1, 0, 0, 0, 903, 904, 7, 6, 0, 0, 904, 905, 7, 13, 0, 0, 905, 906, 7, 1, 0, 0, 906, 907, 7, 18, 0, 0, 907, 908, 7, 3, 0, 0, 908, 143, 1, 0, 0, 0, 909, 910, 5, 41, 0, 0, 910, 145, 1, 0, 0, 0, 911, 912, 7, 5, 0, 0, 912, 913, 7, 6, 0, 0, 913, 914, 7, 20, 0, 0, 914, 915, 7, 3, 0, 0, 915, 147, 1, 0, 0, 0, 916, 917, 5, 61, 0, 0, 917, 918, 5, 61, 0, 0, 918, 149, 1, 0, 0, 0, 919, 920, 5, 61, 0, 0, 920, 921, 5, 126, 0, 0, 921, 151, 1, 0, 0, 0, 922, 923, 5, 33, 0, 0, 923, 924, 5, 61, 0, 0, 924, 153, 1, 0, 0, 0, 925, 926, 5, 60, 0, 0, 926, 155, 1, 0, 0, 0, 927, 928, 5, 60, 0, 0, 928, 929, 5, 61, 0, 0, 929, 157, 1, 0, 0, 0, 930, 931, 5, 62, 0, 0, 931, 159, 1, 0, 0, 0, 932, 933, 5, 62, 0, 0, 933, 934, 5, 61, 0, 0, 934, 161, 1, 0, 0, 0, 935, 936, 5, 43, 0, 0, 936, 163, 1, 0, 0, 0, 937, 938, 5, 45, 0, 0, 938, 165, 1, 0, 0, 0, 939, 940, 5, 42, 0, 0, 940, 167, 1, 0, 0, 0, 941, 942, 5, 47, 0, 0, 942, 169, 1, 0, 0, 0, 943, 944, 5, 37, 0, 0, 944, 171, 1, 0, 0, 0, 945, 946, 5, 123, 0, 0, 946, 173, 1, 0, 0, 0, 947, 948, 5, 125, 0, 0, 948, 175, 1, 0, 0, 0, 949, 950, 3, 46, 15, 0, 950, 951, 1, 0, 0, 0, 951, 952, 6, 80, 13, 0, 952, 177, 1, 0, 0, 0, 953, 956, 3, 140, 62, 0, 954, 957, 3, 76, 30, 0, 955, 957, 3, 90, 37, 0, 956, 954, 1, 0, 0, 0, 956, 955, 1, 0, 0, 0, 957, 961, 1, 0, 0, 0, 958, 960, 3, 92, 38, 0, 959, 958, 1, 0, 0, 0, 960, 963, 1, 0, 0, 0, 961, 959, 1, 0, 0, 0, 961, 962, 1, 0, 0, 0, 962, 971, 1, 0, 0, 0, 963, 961, 1, 0, 0, 0, 964, 966, 3, 140, 62, 0, 965, 967, 3, 74, 29, 0, 966, 965, 1, 0, 0, 0, 967, 968, 1, 0, 0, 0, 968, 966, 1, 0, 0, 0, 968, 969, 1, 0, 0, 0, 969, 971, 1, 0, 0, 0, 970, 953, 1, 0, 0, 0, 970, 964, 1, 0, 0, 0, 971, 179, 1, 0, 0, 0, 972, 973, 5, 91, 0, 0, 973, 974, 1, 0, 0, 0, 974, 975, 6, 82, 0, 0, 975, 976, 6, 82, 0, 0, 976, 181, 1, 0, 0, 0, 977, 978, 5, 93, 0, 0, 978, 979, 1, 0, 0, 0, 979, 980, 6, 83, 12, 0, 980, 981, 6, 83, 12, 0, 981, 183, 1, 0, 0, 0, 982, 986, 3, 76, 30, 0, 983, 985, 3, 92, 38, 0, 984, 983, 1, 0, 0, 0, 985, 988, 1, 0, 0, 0, 986, 984, 1, 0, 0, 0, 986, 987, 1, 0, 0, 0, 987, 999, 1, 0, 0, 0, 988, 986, 1, 0, 0, 0, 989, 992, 3, 90, 37, 0, 990, 992, 3, 84, 34, 0, 991, 989, 1, 0, 0, 0, 991, 990, 1, 0, 0, 0, 992, 994, 1, 0, 0, 0, 993, 995, 3, 92, 38, 0, 994, 993, 1, 0, 0, 0, 995, 996, 1, 0, 0, 0, 996, 994, 1, 0, 0, 0, 996, 997, 1, 0, 0, 0, 997, 999, 1, 0, 0, 0, 998, 982, 1, 0, 0, 0, 998, 991, 1, 0, 0, 0, 999, 185, 1, 0, 0, 0, 1000, 1002, 3, 86, 35, 0, 1001, 1003, 3, 88, 36, 0, 1002, 1001, 1, 0, 0, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1002, 1, 0, 0, 0, 1004, 1005, 1, 0, 0, 0, 1005, 1006, 1, 0, 0, 0, 1006, 1007, 3, 86, 35, 0, 1007, 187, 1, 0, 0, 0, 1008, 1009, 3, 186, 85, 0, 1009, 189, 1, 0, 0, 0, 1010, 1011, 3, 66, 25, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 87, 11, 0, 1013, 191, 1, 0, 0, 0, 1014, 1015, 3, 68, 26, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 88, 11, 0, 1017, 193, 1, 0, 0, 0, 1018, 1019, 3, 70, 27, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 89, 11, 0, 1021, 195, 1, 0, 0, 0, 1022, 1023, 3, 180, 82, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 90, 14, 0, 1025, 1026, 6, 90, 15, 0, 1026, 197, 1, 0, 0, 0, 1027, 1028, 3, 72, 28, 0, 1028, 1029, 1, 0, 0, 0, 1029, 1030, 6, 91, 16, 0, 1030, 1031, 6, 91, 12, 0, 1031, 199, 1, 0, 0, 0, 1032, 1033, 3, 70, 27, 0, 1033, 1034, 1, 0, 0, 0, 1034, 1035, 6, 92, 11, 0, 1035, 201, 1, 0, 0, 0, 1036, 1037, 3, 66, 25, 0, 1037, 1038, 1, 0, 0, 0, 1038, 1039, 6, 93, 11, 0, 1039, 203, 1, 0, 0, 0, 1040, 1041, 3, 68, 26, 0, 1041, 1042, 1, 0, 0, 0, 1042, 1043, 6, 94, 11, 0, 1043, 205, 1, 0, 0, 0, 1044, 1045, 3, 72, 28, 0, 1045, 1046, 1, 0, 0, 0, 1046, 1047, 6, 95, 16, 0, 1047, 1048, 6, 95, 12, 0, 1048, 207, 1, 0, 0, 0, 1049, 1050, 3, 180, 82, 0, 1050, 1051, 1, 0, 0, 0, 1051, 1052, 6, 96, 14, 0, 1052, 209, 1, 0, 0, 0, 1053, 1054, 3, 182, 83, 0, 1054, 1055, 1, 0, 0, 0, 1055, 1056, 6, 97, 17, 0, 1056, 211, 1, 0, 0, 0, 1057, 1058, 3, 110, 47, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1060, 6, 98, 18, 0, 1060, 213, 1, 0, 0, 0, 1061, 1062, 3, 112, 48, 0, 1062, 1063, 1, 0, 0, 0, 1063, 1064, 6, 99, 19, 0, 1064, 215, 1, 0, 0, 0, 1065, 1066, 3, 106, 45, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1068, 6, 100, 20, 0, 1068, 217, 1, 0, 0, 0, 1069, 1070, 7, 16, 0, 0, 1070, 1071, 7, 3, 0, 0, 1071, 1072, 7, 5, 0, 0, 1072, 1073, 7, 12, 0, 0, 1073, 1074, 7, 0, 0, 0, 1074, 1075, 7, 12, 0, 0, 1075, 1076, 7, 5, 0, 0, 1076, 1077, 7, 12, 0, 0, 1077, 219, 1, 0, 0, 0, 1078, 1082, 8, 33, 0, 0, 1079, 1080, 5, 47, 0, 0, 1080, 1082, 8, 34, 0, 0, 1081, 1078, 1, 0, 0, 0, 1081, 1079, 1, 0, 0, 0, 1082, 221, 1, 0, 0, 0, 1083, 1085, 3, 220, 102, 0, 1084, 1083, 1, 0, 0, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1084, 1, 0, 0, 0, 1086, 1087, 1, 0, 0, 0, 1087, 223, 1, 0, 0, 0, 1088, 1089, 3, 222, 103, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 104, 21, 0, 1091, 225, 1, 0, 0, 0, 1092, 1093, 3, 94, 39, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 105, 22, 0, 1095, 227, 1, 0, 0, 0, 1096, 1097, 3, 66, 25, 0, 1097, 1098, 1, 0, 0, 0, 1098, 1099, 6, 106, 11, 0, 1099, 229, 1, 0, 0, 0, 1100, 1101, 3, 68, 26, 0, 1101, 1102, 1, 0, 0, 0, 1102, 1103, 6, 107, 11, 0, 1103, 231, 1, 0, 0, 0, 1104, 1105, 3, 70, 27, 0, 1105, 1106, 1, 0, 0, 0, 1106, 1107, 6, 108, 11, 0, 1107, 233, 1, 0, 0, 0, 1108, 1109, 3, 72, 28, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 109, 16, 0, 1111, 1112, 6, 109, 12, 0, 1112, 235, 1, 0, 0, 0, 1113, 1114, 3, 116, 50, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 110, 23, 0, 1116, 237, 1, 0, 0, 0, 1117, 1118, 3, 112, 48, 0, 1118, 1119, 1, 0, 0, 0, 1119, 1120, 6, 111, 19, 0, 1120, 239, 1, 0, 0, 0, 1121, 1122, 4, 112, 8, 0, 1122, 1123, 3, 140, 62, 0, 1123, 1124, 1, 0, 0, 0, 1124, 1125, 6, 112, 24, 0, 1125, 241, 1, 0, 0, 0, 1126, 1127, 4, 113, 9, 0, 1127, 1128, 3, 178, 81, 0, 1128, 1129, 1, 0, 0, 0, 1129, 1130, 6, 113, 25, 0, 1130, 243, 1, 0, 0, 0, 1131, 1136, 3, 76, 30, 0, 1132, 1136, 3, 74, 29, 0, 1133, 1136, 3, 90, 37, 0, 1134, 1136, 3, 166, 75, 0, 1135, 1131, 1, 0, 0, 0, 1135, 1132, 1, 0, 0, 0, 1135, 1133, 1, 0, 0, 0, 1135, 1134, 1, 0, 0, 0, 1136, 245, 1, 0, 0, 0, 1137, 1140, 3, 76, 30, 0, 1138, 1140, 3, 166, 75, 0, 1139, 1137, 1, 0, 0, 0, 1139, 1138, 1, 0, 0, 0, 1140, 1144, 1, 0, 0, 0, 1141, 1143, 3, 244, 114, 0, 1142, 1141, 1, 0, 0, 0, 1143, 1146, 1, 0, 0, 0, 1144, 1142, 1, 0, 0, 0, 1144, 1145, 1, 0, 0, 0, 1145, 1157, 1, 0, 0, 0, 1146, 1144, 1, 0, 0, 0, 1147, 1150, 3, 90, 37, 0, 1148, 1150, 3, 84, 34, 0, 1149, 1147, 1, 0, 0, 0, 1149, 1148, 1, 0, 0, 0, 1150, 1152, 1, 0, 0, 0, 1151, 1153, 3, 244, 114, 0, 1152, 1151, 1, 0, 0, 0, 1153, 1154, 1, 0, 0, 0, 1154, 1152, 1, 0, 0, 0, 1154, 1155, 1, 0, 0, 0, 1155, 1157, 1, 0, 0, 0, 1156, 1139, 1, 0, 0, 0, 1156, 1149, 1, 0, 0, 0, 1157, 247, 1, 0, 0, 0, 1158, 1161, 3, 246, 115, 0, 1159, 1161, 3, 186, 85, 0, 1160, 1158, 1, 0, 0, 0, 1160, 1159, 1, 0, 0, 0, 1161, 1162, 1, 0, 0, 0, 1162, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 249, 1, 0, 0, 0, 1164, 1165, 3, 66, 25, 0, 1165, 1166, 1, 0, 0, 0, 1166, 1167, 6, 117, 11, 0, 1167, 251, 1, 0, 0, 0, 1168, 1169, 3, 68, 26, 0, 1169, 1170, 1, 0, 0, 0, 1170, 1171, 6, 118, 11, 0, 1171, 253, 1, 0, 0, 0, 1172, 1173, 3, 70, 27, 0, 1173, 1174, 1, 0, 0, 0, 1174, 1175, 6, 119, 11, 0, 1175, 255, 1, 0, 0, 0, 1176, 1177, 3, 72, 28, 0, 1177, 1178, 1, 0, 0, 0, 1178, 1179, 6, 120, 16, 0, 1179, 1180, 6, 120, 12, 0, 1180, 257, 1, 0, 0, 0, 1181, 1182, 3, 106, 45, 0, 1182, 1183, 1, 0, 0, 0, 1183, 1184, 6, 121, 20, 0, 1184, 259, 1, 0, 0, 0, 1185, 1186, 3, 112, 48, 0, 1186, 1187, 1, 0, 0, 0, 1187, 1188, 6, 122, 19, 0, 1188, 261, 1, 0, 0, 0, 1189, 1190, 3, 116, 50, 0, 1190, 1191, 1, 0, 0, 0, 1191, 1192, 6, 123, 23, 0, 1192, 263, 1, 0, 0, 0, 1193, 1194, 4, 124, 10, 0, 1194, 1195, 3, 140, 62, 0, 1195, 1196, 1, 0, 0, 0, 1196, 1197, 6, 124, 24, 0, 1197, 265, 1, 0, 0, 0, 1198, 1199, 4, 125, 11, 0, 1199, 1200, 3, 178, 81, 0, 1200, 1201, 1, 0, 0, 0, 1201, 1202, 6, 125, 25, 0, 1202, 267, 1, 0, 0, 0, 1203, 1204, 7, 12, 0, 0, 1204, 1205, 7, 2, 0, 0, 1205, 269, 1, 0, 0, 0, 1206, 1207, 3, 248, 116, 0, 1207, 1208, 1, 0, 0, 0, 1208, 1209, 6, 127, 26, 0, 1209, 271, 1, 0, 0, 0, 1210, 1211, 3, 66, 25, 0, 1211, 1212, 1, 0, 0, 0, 1212, 1213, 6, 128, 11, 0, 1213, 273, 1, 0, 0, 0, 1214, 1215, 3, 68, 26, 0, 1215, 1216, 1, 0, 0, 0, 1216, 1217, 6, 129, 11, 0, 1217, 275, 1, 0, 0, 0, 1218, 1219, 3, 70, 27, 0, 1219, 1220, 1, 0, 0, 0, 1220, 1221, 6, 130, 11, 0, 1221, 277, 1, 0, 0, 0, 1222, 1223, 3, 72, 28, 0, 1223, 1224, 1, 0, 0, 0, 1224, 1225, 6, 131, 16, 0, 1225, 1226, 6, 131, 12, 0, 1226, 279, 1, 0, 0, 0, 1227, 1228, 3, 180, 82, 0, 1228, 1229, 1, 0, 0, 0, 1229, 1230, 6, 132, 14, 0, 1230, 1231, 6, 132, 27, 0, 1231, 281, 1, 0, 0, 0, 1232, 1233, 7, 7, 0, 0, 1233, 1234, 7, 9, 0, 0, 1234, 1235, 1, 0, 0, 0, 1235, 1236, 6, 133, 28, 0, 1236, 283, 1, 0, 0, 0, 1237, 1238, 7, 19, 0, 0, 1238, 1239, 7, 1, 0, 0, 1239, 1240, 7, 5, 0, 0, 1240, 1241, 7, 10, 0, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1243, 6, 134, 28, 0, 1243, 285, 1, 0, 0, 0, 1244, 1245, 8, 35, 0, 0, 1245, 287, 1, 0, 0, 0, 1246, 1248, 3, 286, 135, 0, 1247, 1246, 1, 0, 0, 0, 1248, 1249, 1, 0, 0, 0, 1249, 1247, 1, 0, 0, 0, 1249, 1250, 1, 0, 0, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 3, 110, 47, 0, 1252, 1254, 1, 0, 0, 0, 1253, 1247, 1, 0, 0, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1256, 1, 0, 0, 0, 1255, 1257, 3, 286, 135, 0, 1256, 1255, 1, 0, 0, 0, 1257, 1258, 1, 0, 0, 0, 1258, 1256, 1, 0, 0, 0, 1258, 1259, 1, 0, 0, 0, 1259, 289, 1, 0, 0, 0, 1260, 1261, 3, 288, 136, 0, 1261, 1262, 1, 0, 0, 0, 1262, 1263, 6, 137, 29, 0, 1263, 291, 1, 0, 0, 0, 1264, 1265, 3, 66, 25, 0, 1265, 1266, 1, 0, 0, 0, 1266, 1267, 6, 138, 11, 0, 1267, 293, 1, 0, 0, 0, 1268, 1269, 3, 68, 26, 0, 1269, 1270, 1, 0, 0, 0, 1270, 1271, 6, 139, 11, 0, 1271, 295, 1, 0, 0, 0, 1272, 1273, 3, 70, 27, 0, 1273, 1274, 1, 0, 0, 0, 1274, 1275, 6, 140, 11, 0, 1275, 297, 1, 0, 0, 0, 1276, 1277, 3, 72, 28, 0, 1277, 1278, 1, 0, 0, 0, 1278, 1279, 6, 141, 16, 0, 1279, 1280, 6, 141, 12, 0, 1280, 1281, 6, 141, 12, 0, 1281, 299, 1, 0, 0, 0, 1282, 1283, 3, 106, 45, 0, 1283, 1284, 1, 0, 0, 0, 1284, 1285, 6, 142, 20, 0, 1285, 301, 1, 0, 0, 0, 1286, 1287, 3, 112, 48, 0, 1287, 1288, 1, 0, 0, 0, 1288, 1289, 6, 143, 19, 0, 1289, 303, 1, 0, 0, 0, 1290, 1291, 3, 116, 50, 0, 1291, 1292, 1, 0, 0, 0, 1292, 1293, 6, 144, 23, 0, 1293, 305, 1, 0, 0, 0, 1294, 1295, 3, 284, 134, 0, 1295, 1296, 1, 0, 0, 0, 1296, 1297, 6, 145, 30, 0, 1297, 307, 1, 0, 0, 0, 1298, 1299, 3, 248, 116, 0, 1299, 1300, 1, 0, 0, 0, 1300, 1301, 6, 146, 26, 0, 1301, 309, 1, 0, 0, 0, 1302, 1303, 3, 188, 86, 0, 1303, 1304, 1, 0, 0, 0, 1304, 1305, 6, 147, 31, 0, 1305, 311, 1, 0, 0, 0, 1306, 1307, 4, 148, 12, 0, 1307, 1308, 3, 140, 62, 0, 1308, 1309, 1, 0, 0, 0, 1309, 1310, 6, 148, 24, 0, 1310, 313, 1, 0, 0, 0, 1311, 1312, 4, 149, 13, 0, 1312, 1313, 3, 178, 81, 0, 1313, 1314, 1, 0, 0, 0, 1314, 1315, 6, 149, 25, 0, 1315, 315, 1, 0, 0, 0, 1316, 1317, 3, 66, 25, 0, 1317, 1318, 1, 0, 0, 0, 1318, 1319, 6, 150, 11, 0, 1319, 317, 1, 0, 0, 0, 1320, 1321, 3, 68, 26, 0, 1321, 1322, 1, 0, 0, 0, 1322, 1323, 6, 151, 11, 0, 1323, 319, 1, 0, 0, 0, 1324, 1325, 3, 70, 27, 0, 1325, 1326, 1, 0, 0, 0, 1326, 1327, 6, 152, 11, 0, 1327, 321, 1, 0, 0, 0, 1328, 1329, 3, 72, 28, 0, 1329, 1330, 1, 0, 0, 0, 1330, 1331, 6, 153, 16, 0, 1331, 1332, 6, 153, 12, 0, 1332, 323, 1, 0, 0, 0, 1333, 1334, 3, 116, 50, 0, 1334, 1335, 1, 0, 0, 0, 1335, 1336, 6, 154, 23, 0, 1336, 325, 1, 0, 0, 0, 1337, 1338, 4, 155, 14, 0, 1338, 1339, 3, 140, 62, 0, 1339, 1340, 1, 0, 0, 0, 1340, 1341, 6, 155, 24, 0, 1341, 327, 1, 0, 0, 0, 1342, 1343, 4, 156, 15, 0, 1343, 1344, 3, 178, 81, 0, 1344, 1345, 1, 0, 0, 0, 1345, 1346, 6, 156, 25, 0, 1346, 329, 1, 0, 0, 0, 1347, 1348, 3, 188, 86, 0, 1348, 1349, 1, 0, 0, 0, 1349, 1350, 6, 157, 31, 0, 1350, 331, 1, 0, 0, 0, 1351, 1352, 3, 184, 84, 0, 1352, 1353, 1, 0, 0, 0, 1353, 1354, 6, 158, 32, 0, 1354, 333, 1, 0, 0, 0, 1355, 1356, 3, 66, 25, 0, 1356, 1357, 1, 0, 0, 0, 1357, 1358, 6, 159, 11, 0, 1358, 335, 1, 0, 0, 0, 1359, 1360, 3, 68, 26, 0, 1360, 1361, 1, 0, 0, 0, 1361, 1362, 6, 160, 11, 0, 1362, 337, 1, 0, 0, 0, 1363, 1364, 3, 70, 27, 0, 1364, 1365, 1, 0, 0, 0, 1365, 1366, 6, 161, 11, 0, 1366, 339, 1, 0, 0, 0, 1367, 1368, 3, 72, 28, 0, 1368, 1369, 1, 0, 0, 0, 1369, 1370, 6, 162, 16, 0, 1370, 1371, 6, 162, 12, 0, 1371, 341, 1, 0, 0, 0, 1372, 1373, 7, 1, 0, 0, 1373, 1374, 7, 9, 0, 0, 1374, 1375, 7, 15, 0, 0, 1375, 1376, 7, 7, 0, 0, 1376, 343, 1, 0, 0, 0, 1377, 1378, 3, 66, 25, 0, 1378, 1379, 1, 0, 0, 0, 1379, 1380, 6, 164, 11, 0, 1380, 345, 1, 0, 0, 0, 1381, 1382, 3, 68, 26, 0, 1382, 1383, 1, 0, 0, 0, 1383, 1384, 6, 165, 11, 0, 1384, 347, 1, 0, 0, 0, 1385, 1386, 3, 70, 27, 0, 1386, 1387, 1, 0, 0, 0, 1387, 1388, 6, 166, 11, 0, 1388, 349, 1, 0, 0, 0, 1389, 1390, 3, 182, 83, 0, 1390, 1391, 1, 0, 0, 0, 1391, 1392, 6, 167, 17, 0, 1392, 1393, 6, 167, 12, 0, 1393, 351, 1, 0, 0, 0, 1394, 1395, 3, 110, 47, 0, 1395, 1396, 1, 0, 0, 0, 1396, 1397, 6, 168, 18, 0, 1397, 353, 1, 0, 0, 0, 1398, 1404, 3, 84, 34, 0, 1399, 1404, 3, 74, 29, 0, 1400, 1404, 3, 116, 50, 0, 1401, 1404, 3, 76, 30, 0, 1402, 1404, 3, 90, 37, 0, 1403, 1398, 1, 0, 0, 0, 1403, 1399, 1, 0, 0, 0, 1403, 1400, 1, 0, 0, 0, 1403, 1401, 1, 0, 0, 0, 1403, 1402, 1, 0, 0, 0, 1404, 1405, 1, 0, 0, 0, 1405, 1403, 1, 0, 0, 0, 1405, 1406, 1, 0, 0, 0, 1406, 355, 1, 0, 0, 0, 1407, 1408, 3, 66, 25, 0, 1408, 1409, 1, 0, 0, 0, 1409, 1410, 6, 170, 11, 0, 1410, 357, 1, 0, 0, 0, 1411, 1412, 3, 68, 26, 0, 1412, 1413, 1, 0, 0, 0, 1413, 1414, 6, 171, 11, 0, 1414, 359, 1, 0, 0, 0, 1415, 1416, 3, 70, 27, 0, 1416, 1417, 1, 0, 0, 0, 1417, 1418, 6, 172, 11, 0, 1418, 361, 1, 0, 0, 0, 1419, 1420, 3, 72, 28, 0, 1420, 1421, 1, 0, 0, 0, 1421, 1422, 6, 173, 16, 0, 1422, 1423, 6, 173, 12, 0, 1423, 363, 1, 0, 0, 0, 1424, 1425, 3, 110, 47, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 174, 18, 0, 1427, 365, 1, 0, 0, 0, 1428, 1429, 3, 112, 48, 0, 1429, 1430, 1, 0, 0, 0, 1430, 1431, 6, 175, 19, 0, 1431, 367, 1, 0, 0, 0, 1432, 1433, 3, 116, 50, 0, 1433, 1434, 1, 0, 0, 0, 1434, 1435, 6, 176, 23, 0, 1435, 369, 1, 0, 0, 0, 1436, 1437, 3, 282, 133, 0, 1437, 1438, 1, 0, 0, 0, 1438, 1439, 6, 177, 33, 0, 1439, 1440, 6, 177, 34, 0, 1440, 371, 1, 0, 0, 0, 1441, 1442, 3, 222, 103, 0, 1442, 1443, 1, 0, 0, 0, 1443, 1444, 6, 178, 21, 0, 1444, 373, 1, 0, 0, 0, 1445, 1446, 3, 94, 39, 0, 1446, 1447, 1, 0, 0, 0, 1447, 1448, 6, 179, 22, 0, 1448, 375, 1, 0, 0, 0, 1449, 1450, 3, 66, 25, 0, 1450, 1451, 1, 0, 0, 0, 1451, 1452, 6, 180, 11, 0, 1452, 377, 1, 0, 0, 0, 1453, 1454, 3, 68, 26, 0, 1454, 1455, 1, 0, 0, 0, 1455, 1456, 6, 181, 11, 0, 1456, 379, 1, 0, 0, 0, 1457, 1458, 3, 70, 27, 0, 1458, 1459, 1, 0, 0, 0, 1459, 1460, 6, 182, 11, 0, 1460, 381, 1, 0, 0, 0, 1461, 1462, 3, 72, 28, 0, 1462, 1463, 1, 0, 0, 0, 1463, 1464, 6, 183, 16, 0, 1464, 1465, 6, 183, 12, 0, 1465, 1466, 6, 183, 12, 0, 1466, 383, 1, 0, 0, 0, 1467, 1468, 3, 112, 48, 0, 1468, 1469, 1, 0, 0, 0, 1469, 1470, 6, 184, 19, 0, 1470, 385, 1, 0, 0, 0, 1471, 1472, 3, 116, 50, 0, 1472, 1473, 1, 0, 0, 0, 1473, 1474, 6, 185, 23, 0, 1474, 387, 1, 0, 0, 0, 1475, 1476, 3, 248, 116, 0, 1476, 1477, 1, 0, 0, 0, 1477, 1478, 6, 186, 26, 0, 1478, 389, 1, 0, 0, 0, 1479, 1480, 3, 66, 25, 0, 1480, 1481, 1, 0, 0, 0, 1481, 1482, 6, 187, 11, 0, 1482, 391, 1, 0, 0, 0, 1483, 1484, 3, 68, 26, 0, 1484, 1485, 1, 0, 0, 0, 1485, 1486, 6, 188, 11, 0, 1486, 393, 1, 0, 0, 0, 1487, 1488, 3, 70, 27, 0, 1488, 1489, 1, 0, 0, 0, 1489, 1490, 6, 189, 11, 0, 1490, 395, 1, 0, 0, 0, 1491, 1492, 3, 72, 28, 0, 1492, 1493, 1, 0, 0, 0, 1493, 1494, 6, 190, 16, 0, 1494, 1495, 6, 190, 12, 0, 1495, 397, 1, 0, 0, 0, 1496, 1497, 3, 54, 19, 0, 1497, 1498, 1, 0, 0, 0, 1498, 1499, 6, 191, 35, 0, 1499, 399, 1, 0, 0, 0, 1500, 1501, 3, 268, 126, 0, 1501, 1502, 1, 0, 0, 0, 1502, 1503, 6, 192, 36, 0, 1503, 401, 1, 0, 0, 0, 1504, 1505, 3, 282, 133, 0, 1505, 1506, 1, 0, 0, 0, 1506, 1507, 6, 193, 33, 0, 1507, 1508, 6, 193, 12, 0, 1508, 1509, 6, 193, 0, 0, 1509, 403, 1, 0, 0, 0, 1510, 1511, 7, 20, 0, 0, 1511, 1512, 7, 2, 0, 0, 1512, 1513, 7, 1, 0, 0, 1513, 1514, 7, 9, 0, 0, 1514, 1515, 7, 17, 0, 0, 1515, 1516, 1, 0, 0, 0, 1516, 1517, 6, 194, 12, 0, 1517, 1518, 6, 194, 0, 0, 1518, 405, 1, 0, 0, 0, 1519, 1520, 3, 222, 103, 0, 1520, 1521, 1, 0, 0, 0, 1521, 1522, 6, 195, 21, 0, 1522, 407, 1, 0, 0, 0, 1523, 1524, 3, 94, 39, 0, 1524, 1525, 1, 0, 0, 0, 1525, 1526, 6, 196, 22, 0, 1526, 409, 1, 0, 0, 0, 1527, 1528, 3, 110, 47, 0, 1528, 1529, 1, 0, 0, 0, 1529, 1530, 6, 197, 18, 0, 1530, 411, 1, 0, 0, 0, 1531, 1532, 3, 184, 84, 0, 1532, 1533, 1, 0, 0, 0, 1533, 1534, 6, 198, 32, 0, 1534, 413, 1, 0, 0, 0, 1535, 1536, 3, 188, 86, 0, 1536, 1537, 1, 0, 0, 0, 1537, 1538, 6, 199, 31, 0, 1538, 415, 1, 0, 0, 0, 1539, 1540, 3, 66, 25, 0, 1540, 1541, 1, 0, 0, 0, 1541, 1542, 6, 200, 11, 0, 1542, 417, 1, 0, 0, 0, 1543, 1544, 3, 68, 26, 0, 1544, 1545, 1, 0, 0, 0, 1545, 1546, 6, 201, 11, 0, 1546, 419, 1, 0, 0, 0, 1547, 1548, 3, 70, 27, 0, 1548, 1549, 1, 0, 0, 0, 1549, 1550, 6, 202, 11, 0, 1550, 421, 1, 0, 0, 0, 1551, 1552, 3, 72, 28, 0, 1552, 1553, 1, 0, 0, 0, 1553, 1554, 6, 203, 16, 0, 1554, 1555, 6, 203, 12, 0, 1555, 423, 1, 0, 0, 0, 1556, 1557, 3, 222, 103, 0, 1557, 1558, 1, 0, 0, 0, 1558, 1559, 6, 204, 21, 0, 1559, 1560, 6, 204, 12, 0, 1560, 1561, 6, 204, 37, 0, 1561, 425, 1, 0, 0, 0, 1562, 1563, 3, 94, 39, 0, 1563, 1564, 1, 0, 0, 0, 1564, 1565, 6, 205, 22, 0, 1565, 1566, 6, 205, 12, 0, 1566, 1567, 6, 205, 37, 0, 1567, 427, 1, 0, 0, 0, 1568, 1569, 3, 66, 25, 0, 1569, 1570, 1, 0, 0, 0, 1570, 1571, 6, 206, 11, 0, 1571, 429, 1, 0, 0, 0, 1572, 1573, 3, 68, 26, 0, 1573, 1574, 1, 0, 0, 0, 1574, 1575, 6, 207, 11, 0, 1575, 431, 1, 0, 0, 0, 1576, 1577, 3, 70, 27, 0, 1577, 1578, 1, 0, 0, 0, 1578, 1579, 6, 208, 11, 0, 1579, 433, 1, 0, 0, 0, 1580, 1581, 3, 110, 47, 0, 1581, 1582, 1, 0, 0, 0, 1582, 1583, 6, 209, 18, 0, 1583, 1584, 6, 209, 12, 0, 1584, 1585, 6, 209, 9, 0, 1585, 435, 1, 0, 0, 0, 1586, 1587, 3, 112, 48, 0, 1587, 1588, 1, 0, 0, 0, 1588, 1589, 6, 210, 19, 0, 1589, 1590, 6, 210, 12, 0, 1590, 1591, 6, 210, 9, 0, 1591, 437, 1, 0, 0, 0, 1592, 1593, 3, 66, 25, 0, 1593, 1594, 1, 0, 0, 0, 1594, 1595, 6, 211, 11, 0, 1595, 439, 1, 0, 0, 0, 1596, 1597, 3, 68, 26, 0, 1597, 1598, 1, 0, 0, 0, 1598, 1599, 6, 212, 11, 0, 1599, 441, 1, 0, 0, 0, 1600, 1601, 3, 70, 27, 0, 1601, 1602, 1, 0, 0, 0, 1602, 1603, 6, 213, 11, 0, 1603, 443, 1, 0, 0, 0, 1604, 1605, 3, 188, 86, 0, 1605, 1606, 1, 0, 0, 0, 1606, 1607, 6, 214, 12, 0, 1607, 1608, 6, 214, 0, 0, 1608, 1609, 6, 214, 31, 0, 1609, 445, 1, 0, 0, 0, 1610, 1611, 3, 184, 84, 0, 1611, 1612, 1, 0, 0, 0, 1612, 1613, 6, 215, 12, 0, 1613, 1614, 6, 215, 0, 0, 1614, 1615, 6, 215, 32, 0, 1615, 447, 1, 0, 0, 0, 1616, 1617, 3, 100, 42, 0, 1617, 1618, 1, 0, 0, 0, 1618, 1619, 6, 216, 12, 0, 1619, 1620, 6, 216, 0, 0, 1620, 1621, 6, 216, 38, 0, 1621, 449, 1, 0, 0, 0, 1622, 1623, 3, 72, 28, 0, 1623, 1624, 1, 0, 0, 0, 1624, 1625, 6, 217, 16, 0, 1625, 1626, 6, 217, 12, 0, 1626, 451, 1, 0, 0, 0, 66, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 665, 675, 679, 682, 691, 693, 704, 723, 728, 737, 744, 749, 751, 762, 770, 773, 775, 780, 785, 791, 798, 803, 809, 812, 820, 824, 956, 961, 968, 970, 986, 991, 996, 998, 1004, 1081, 1086, 1135, 1139, 1144, 1149, 1154, 1156, 1160, 1162, 1249, 1253, 1258, 1403, 1405, 39, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 14, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 72, 0, 5, 0, 0, 7, 29, 0, 7, 73, 0, 7, 38, 0, 7, 39, 0, 7, 36, 0, 7, 83, 0, 7, 30, 0, 7, 41, 0, 7, 53, 0, 7, 71, 0, 7, 87, 0, 5, 10, 0, 5, 7, 0, 7, 97, 0, 7, 96, 0, 7, 75, 0, 7, 74, 0, 7, 95, 0, 5, 12, 0, 7, 20, 0, 7, 91, 0, 5, 15, 0, 7, 33, 0] \ No newline at end of file +[4, 0, 130, 1609, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 2, 198, 7, 198, 2, 199, 7, 199, 2, 200, 7, 200, 2, 201, 7, 201, 2, 202, 7, 202, 2, 203, 7, 203, 2, 204, 7, 204, 2, 205, 7, 205, 2, 206, 7, 206, 2, 207, 7, 207, 2, 208, 7, 208, 2, 209, 7, 209, 2, 210, 7, 210, 2, 211, 7, 211, 2, 212, 7, 212, 2, 213, 7, 213, 2, 214, 7, 214, 2, 215, 7, 215, 2, 216, 7, 216, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 653, 8, 23, 11, 23, 12, 23, 654, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 663, 8, 24, 10, 24, 12, 24, 666, 9, 24, 1, 24, 3, 24, 669, 8, 24, 1, 24, 3, 24, 672, 8, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 681, 8, 25, 10, 25, 12, 25, 684, 9, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 4, 26, 692, 8, 26, 11, 26, 12, 26, 693, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 3, 32, 713, 8, 32, 1, 32, 4, 32, 716, 8, 32, 11, 32, 12, 32, 717, 1, 33, 1, 33, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 3, 35, 727, 8, 35, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 3, 37, 734, 8, 37, 1, 38, 1, 38, 1, 38, 5, 38, 739, 8, 38, 10, 38, 12, 38, 742, 9, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 750, 8, 38, 10, 38, 12, 38, 753, 9, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 760, 8, 38, 1, 38, 3, 38, 763, 8, 38, 3, 38, 765, 8, 38, 1, 39, 4, 39, 768, 8, 39, 11, 39, 12, 39, 769, 1, 40, 4, 40, 773, 8, 40, 11, 40, 12, 40, 774, 1, 40, 1, 40, 5, 40, 779, 8, 40, 10, 40, 12, 40, 782, 9, 40, 1, 40, 1, 40, 4, 40, 786, 8, 40, 11, 40, 12, 40, 787, 1, 40, 4, 40, 791, 8, 40, 11, 40, 12, 40, 792, 1, 40, 1, 40, 5, 40, 797, 8, 40, 10, 40, 12, 40, 800, 9, 40, 3, 40, 802, 8, 40, 1, 40, 1, 40, 1, 40, 1, 40, 4, 40, 808, 8, 40, 11, 40, 12, 40, 809, 1, 40, 1, 40, 3, 40, 814, 8, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 3, 80, 946, 8, 80, 1, 80, 5, 80, 949, 8, 80, 10, 80, 12, 80, 952, 9, 80, 1, 80, 1, 80, 4, 80, 956, 8, 80, 11, 80, 12, 80, 957, 3, 80, 960, 8, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 5, 83, 974, 8, 83, 10, 83, 12, 83, 977, 9, 83, 1, 83, 1, 83, 3, 83, 981, 8, 83, 1, 83, 4, 83, 984, 8, 83, 11, 83, 12, 83, 985, 3, 83, 988, 8, 83, 1, 84, 1, 84, 4, 84, 992, 8, 84, 11, 84, 12, 84, 993, 1, 84, 1, 84, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 100, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 3, 101, 1071, 8, 101, 1, 102, 4, 102, 1074, 8, 102, 11, 102, 12, 102, 1075, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 3, 113, 1123, 8, 113, 1, 114, 1, 114, 3, 114, 1127, 8, 114, 1, 114, 5, 114, 1130, 8, 114, 10, 114, 12, 114, 1133, 9, 114, 1, 114, 1, 114, 3, 114, 1137, 8, 114, 1, 114, 4, 114, 1140, 8, 114, 11, 114, 12, 114, 1141, 3, 114, 1144, 8, 114, 1, 115, 1, 115, 4, 115, 1148, 8, 115, 11, 115, 12, 115, 1149, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 135, 4, 135, 1233, 8, 135, 11, 135, 12, 135, 1234, 1, 135, 1, 135, 3, 135, 1239, 8, 135, 1, 135, 4, 135, 1242, 8, 135, 11, 135, 12, 135, 1243, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 4, 168, 1385, 8, 168, 11, 168, 12, 168, 1386, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 198, 1, 198, 1, 198, 1, 198, 1, 199, 1, 199, 1, 199, 1, 199, 1, 200, 1, 200, 1, 200, 1, 200, 1, 201, 1, 201, 1, 201, 1, 201, 1, 202, 1, 202, 1, 202, 1, 202, 1, 202, 1, 203, 1, 203, 1, 203, 1, 203, 1, 203, 1, 203, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 204, 1, 205, 1, 205, 1, 205, 1, 205, 1, 206, 1, 206, 1, 206, 1, 206, 1, 207, 1, 207, 1, 207, 1, 207, 1, 208, 1, 208, 1, 208, 1, 208, 1, 208, 1, 208, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 210, 1, 210, 1, 210, 1, 210, 1, 211, 1, 211, 1, 211, 1, 211, 1, 212, 1, 212, 1, 212, 1, 212, 1, 213, 1, 213, 1, 213, 1, 213, 1, 213, 1, 213, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 215, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 2, 682, 751, 0, 217, 16, 1, 18, 2, 20, 3, 22, 4, 24, 5, 26, 6, 28, 7, 30, 8, 32, 9, 34, 10, 36, 11, 38, 12, 40, 13, 42, 14, 44, 15, 46, 16, 48, 17, 50, 18, 52, 19, 54, 20, 56, 21, 58, 22, 60, 23, 62, 24, 64, 25, 66, 26, 68, 27, 70, 28, 72, 0, 74, 0, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 0, 88, 0, 90, 0, 92, 29, 94, 30, 96, 31, 98, 32, 100, 33, 102, 34, 104, 35, 106, 36, 108, 37, 110, 38, 112, 39, 114, 40, 116, 41, 118, 42, 120, 43, 122, 44, 124, 45, 126, 46, 128, 47, 130, 48, 132, 49, 134, 50, 136, 51, 138, 52, 140, 53, 142, 54, 144, 55, 146, 56, 148, 57, 150, 58, 152, 59, 154, 60, 156, 61, 158, 62, 160, 63, 162, 64, 164, 65, 166, 66, 168, 67, 170, 68, 172, 69, 174, 0, 176, 70, 178, 71, 180, 72, 182, 73, 184, 0, 186, 74, 188, 75, 190, 76, 192, 77, 194, 0, 196, 0, 198, 78, 200, 79, 202, 80, 204, 0, 206, 0, 208, 0, 210, 0, 212, 0, 214, 0, 216, 81, 218, 0, 220, 82, 222, 0, 224, 0, 226, 83, 228, 84, 230, 85, 232, 0, 234, 0, 236, 0, 238, 0, 240, 0, 242, 0, 244, 0, 246, 86, 248, 87, 250, 88, 252, 89, 254, 0, 256, 0, 258, 0, 260, 0, 262, 0, 264, 0, 266, 90, 268, 0, 270, 91, 272, 92, 274, 93, 276, 0, 278, 0, 280, 94, 282, 95, 284, 0, 286, 96, 288, 0, 290, 97, 292, 98, 294, 99, 296, 0, 298, 0, 300, 0, 302, 0, 304, 0, 306, 0, 308, 0, 310, 0, 312, 0, 314, 100, 316, 101, 318, 102, 320, 0, 322, 0, 324, 0, 326, 0, 328, 0, 330, 0, 332, 103, 334, 104, 336, 105, 338, 0, 340, 106, 342, 107, 344, 108, 346, 109, 348, 0, 350, 0, 352, 110, 354, 111, 356, 112, 358, 113, 360, 0, 362, 0, 364, 0, 366, 0, 368, 0, 370, 0, 372, 0, 374, 114, 376, 115, 378, 116, 380, 0, 382, 0, 384, 0, 386, 0, 388, 117, 390, 118, 392, 119, 394, 0, 396, 120, 398, 0, 400, 0, 402, 121, 404, 0, 406, 0, 408, 0, 410, 0, 412, 0, 414, 122, 416, 123, 418, 124, 420, 0, 422, 0, 424, 0, 426, 125, 428, 126, 430, 127, 432, 0, 434, 0, 436, 128, 438, 129, 440, 130, 442, 0, 444, 0, 446, 0, 448, 0, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 36, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 2, 0, 74, 74, 106, 106, 1636, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 66, 1, 0, 0, 0, 0, 68, 1, 0, 0, 0, 1, 70, 1, 0, 0, 0, 1, 92, 1, 0, 0, 0, 1, 94, 1, 0, 0, 0, 1, 96, 1, 0, 0, 0, 1, 98, 1, 0, 0, 0, 1, 100, 1, 0, 0, 0, 1, 102, 1, 0, 0, 0, 1, 104, 1, 0, 0, 0, 1, 106, 1, 0, 0, 0, 1, 108, 1, 0, 0, 0, 1, 110, 1, 0, 0, 0, 1, 112, 1, 0, 0, 0, 1, 114, 1, 0, 0, 0, 1, 116, 1, 0, 0, 0, 1, 118, 1, 0, 0, 0, 1, 120, 1, 0, 0, 0, 1, 122, 1, 0, 0, 0, 1, 124, 1, 0, 0, 0, 1, 126, 1, 0, 0, 0, 1, 128, 1, 0, 0, 0, 1, 130, 1, 0, 0, 0, 1, 132, 1, 0, 0, 0, 1, 134, 1, 0, 0, 0, 1, 136, 1, 0, 0, 0, 1, 138, 1, 0, 0, 0, 1, 140, 1, 0, 0, 0, 1, 142, 1, 0, 0, 0, 1, 144, 1, 0, 0, 0, 1, 146, 1, 0, 0, 0, 1, 148, 1, 0, 0, 0, 1, 150, 1, 0, 0, 0, 1, 152, 1, 0, 0, 0, 1, 154, 1, 0, 0, 0, 1, 156, 1, 0, 0, 0, 1, 158, 1, 0, 0, 0, 1, 160, 1, 0, 0, 0, 1, 162, 1, 0, 0, 0, 1, 164, 1, 0, 0, 0, 1, 166, 1, 0, 0, 0, 1, 168, 1, 0, 0, 0, 1, 170, 1, 0, 0, 0, 1, 172, 1, 0, 0, 0, 1, 174, 1, 0, 0, 0, 1, 176, 1, 0, 0, 0, 1, 178, 1, 0, 0, 0, 1, 180, 1, 0, 0, 0, 1, 182, 1, 0, 0, 0, 1, 186, 1, 0, 0, 0, 1, 188, 1, 0, 0, 0, 1, 190, 1, 0, 0, 0, 1, 192, 1, 0, 0, 0, 2, 194, 1, 0, 0, 0, 2, 196, 1, 0, 0, 0, 2, 198, 1, 0, 0, 0, 2, 200, 1, 0, 0, 0, 2, 202, 1, 0, 0, 0, 3, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 208, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 3, 214, 1, 0, 0, 0, 3, 216, 1, 0, 0, 0, 3, 220, 1, 0, 0, 0, 3, 222, 1, 0, 0, 0, 3, 224, 1, 0, 0, 0, 3, 226, 1, 0, 0, 0, 3, 228, 1, 0, 0, 0, 3, 230, 1, 0, 0, 0, 4, 232, 1, 0, 0, 0, 4, 234, 1, 0, 0, 0, 4, 236, 1, 0, 0, 0, 4, 238, 1, 0, 0, 0, 4, 240, 1, 0, 0, 0, 4, 246, 1, 0, 0, 0, 4, 248, 1, 0, 0, 0, 4, 250, 1, 0, 0, 0, 4, 252, 1, 0, 0, 0, 5, 254, 1, 0, 0, 0, 5, 256, 1, 0, 0, 0, 5, 258, 1, 0, 0, 0, 5, 260, 1, 0, 0, 0, 5, 262, 1, 0, 0, 0, 5, 264, 1, 0, 0, 0, 5, 266, 1, 0, 0, 0, 5, 268, 1, 0, 0, 0, 5, 270, 1, 0, 0, 0, 5, 272, 1, 0, 0, 0, 5, 274, 1, 0, 0, 0, 6, 276, 1, 0, 0, 0, 6, 278, 1, 0, 0, 0, 6, 280, 1, 0, 0, 0, 6, 282, 1, 0, 0, 0, 6, 286, 1, 0, 0, 0, 6, 288, 1, 0, 0, 0, 6, 290, 1, 0, 0, 0, 6, 292, 1, 0, 0, 0, 6, 294, 1, 0, 0, 0, 7, 296, 1, 0, 0, 0, 7, 298, 1, 0, 0, 0, 7, 300, 1, 0, 0, 0, 7, 302, 1, 0, 0, 0, 7, 304, 1, 0, 0, 0, 7, 306, 1, 0, 0, 0, 7, 308, 1, 0, 0, 0, 7, 310, 1, 0, 0, 0, 7, 312, 1, 0, 0, 0, 7, 314, 1, 0, 0, 0, 7, 316, 1, 0, 0, 0, 7, 318, 1, 0, 0, 0, 8, 320, 1, 0, 0, 0, 8, 322, 1, 0, 0, 0, 8, 324, 1, 0, 0, 0, 8, 326, 1, 0, 0, 0, 8, 328, 1, 0, 0, 0, 8, 330, 1, 0, 0, 0, 8, 332, 1, 0, 0, 0, 8, 334, 1, 0, 0, 0, 8, 336, 1, 0, 0, 0, 9, 338, 1, 0, 0, 0, 9, 340, 1, 0, 0, 0, 9, 342, 1, 0, 0, 0, 9, 344, 1, 0, 0, 0, 9, 346, 1, 0, 0, 0, 10, 348, 1, 0, 0, 0, 10, 350, 1, 0, 0, 0, 10, 352, 1, 0, 0, 0, 10, 354, 1, 0, 0, 0, 10, 356, 1, 0, 0, 0, 10, 358, 1, 0, 0, 0, 11, 360, 1, 0, 0, 0, 11, 362, 1, 0, 0, 0, 11, 364, 1, 0, 0, 0, 11, 366, 1, 0, 0, 0, 11, 368, 1, 0, 0, 0, 11, 370, 1, 0, 0, 0, 11, 372, 1, 0, 0, 0, 11, 374, 1, 0, 0, 0, 11, 376, 1, 0, 0, 0, 11, 378, 1, 0, 0, 0, 12, 380, 1, 0, 0, 0, 12, 382, 1, 0, 0, 0, 12, 384, 1, 0, 0, 0, 12, 386, 1, 0, 0, 0, 12, 388, 1, 0, 0, 0, 12, 390, 1, 0, 0, 0, 12, 392, 1, 0, 0, 0, 13, 394, 1, 0, 0, 0, 13, 396, 1, 0, 0, 0, 13, 398, 1, 0, 0, 0, 13, 400, 1, 0, 0, 0, 13, 402, 1, 0, 0, 0, 13, 404, 1, 0, 0, 0, 13, 406, 1, 0, 0, 0, 13, 408, 1, 0, 0, 0, 13, 410, 1, 0, 0, 0, 13, 412, 1, 0, 0, 0, 13, 414, 1, 0, 0, 0, 13, 416, 1, 0, 0, 0, 13, 418, 1, 0, 0, 0, 14, 420, 1, 0, 0, 0, 14, 422, 1, 0, 0, 0, 14, 424, 1, 0, 0, 0, 14, 426, 1, 0, 0, 0, 14, 428, 1, 0, 0, 0, 14, 430, 1, 0, 0, 0, 15, 432, 1, 0, 0, 0, 15, 434, 1, 0, 0, 0, 15, 436, 1, 0, 0, 0, 15, 438, 1, 0, 0, 0, 15, 440, 1, 0, 0, 0, 15, 442, 1, 0, 0, 0, 15, 444, 1, 0, 0, 0, 15, 446, 1, 0, 0, 0, 15, 448, 1, 0, 0, 0, 16, 450, 1, 0, 0, 0, 18, 460, 1, 0, 0, 0, 20, 467, 1, 0, 0, 0, 22, 476, 1, 0, 0, 0, 24, 483, 1, 0, 0, 0, 26, 493, 1, 0, 0, 0, 28, 500, 1, 0, 0, 0, 30, 507, 1, 0, 0, 0, 32, 514, 1, 0, 0, 0, 34, 522, 1, 0, 0, 0, 36, 534, 1, 0, 0, 0, 38, 543, 1, 0, 0, 0, 40, 549, 1, 0, 0, 0, 42, 556, 1, 0, 0, 0, 44, 563, 1, 0, 0, 0, 46, 571, 1, 0, 0, 0, 48, 579, 1, 0, 0, 0, 50, 588, 1, 0, 0, 0, 52, 603, 1, 0, 0, 0, 54, 615, 1, 0, 0, 0, 56, 626, 1, 0, 0, 0, 58, 634, 1, 0, 0, 0, 60, 642, 1, 0, 0, 0, 62, 652, 1, 0, 0, 0, 64, 658, 1, 0, 0, 0, 66, 675, 1, 0, 0, 0, 68, 691, 1, 0, 0, 0, 70, 697, 1, 0, 0, 0, 72, 701, 1, 0, 0, 0, 74, 703, 1, 0, 0, 0, 76, 705, 1, 0, 0, 0, 78, 708, 1, 0, 0, 0, 80, 710, 1, 0, 0, 0, 82, 719, 1, 0, 0, 0, 84, 721, 1, 0, 0, 0, 86, 726, 1, 0, 0, 0, 88, 728, 1, 0, 0, 0, 90, 733, 1, 0, 0, 0, 92, 764, 1, 0, 0, 0, 94, 767, 1, 0, 0, 0, 96, 813, 1, 0, 0, 0, 98, 815, 1, 0, 0, 0, 100, 818, 1, 0, 0, 0, 102, 822, 1, 0, 0, 0, 104, 826, 1, 0, 0, 0, 106, 828, 1, 0, 0, 0, 108, 831, 1, 0, 0, 0, 110, 833, 1, 0, 0, 0, 112, 835, 1, 0, 0, 0, 114, 840, 1, 0, 0, 0, 116, 842, 1, 0, 0, 0, 118, 848, 1, 0, 0, 0, 120, 854, 1, 0, 0, 0, 122, 857, 1, 0, 0, 0, 124, 860, 1, 0, 0, 0, 126, 865, 1, 0, 0, 0, 128, 870, 1, 0, 0, 0, 130, 872, 1, 0, 0, 0, 132, 876, 1, 0, 0, 0, 134, 881, 1, 0, 0, 0, 136, 887, 1, 0, 0, 0, 138, 890, 1, 0, 0, 0, 140, 892, 1, 0, 0, 0, 142, 898, 1, 0, 0, 0, 144, 900, 1, 0, 0, 0, 146, 905, 1, 0, 0, 0, 148, 908, 1, 0, 0, 0, 150, 911, 1, 0, 0, 0, 152, 914, 1, 0, 0, 0, 154, 916, 1, 0, 0, 0, 156, 919, 1, 0, 0, 0, 158, 921, 1, 0, 0, 0, 160, 924, 1, 0, 0, 0, 162, 926, 1, 0, 0, 0, 164, 928, 1, 0, 0, 0, 166, 930, 1, 0, 0, 0, 168, 932, 1, 0, 0, 0, 170, 934, 1, 0, 0, 0, 172, 936, 1, 0, 0, 0, 174, 938, 1, 0, 0, 0, 176, 959, 1, 0, 0, 0, 178, 961, 1, 0, 0, 0, 180, 966, 1, 0, 0, 0, 182, 987, 1, 0, 0, 0, 184, 989, 1, 0, 0, 0, 186, 997, 1, 0, 0, 0, 188, 999, 1, 0, 0, 0, 190, 1003, 1, 0, 0, 0, 192, 1007, 1, 0, 0, 0, 194, 1011, 1, 0, 0, 0, 196, 1016, 1, 0, 0, 0, 198, 1021, 1, 0, 0, 0, 200, 1025, 1, 0, 0, 0, 202, 1029, 1, 0, 0, 0, 204, 1033, 1, 0, 0, 0, 206, 1038, 1, 0, 0, 0, 208, 1042, 1, 0, 0, 0, 210, 1046, 1, 0, 0, 0, 212, 1050, 1, 0, 0, 0, 214, 1054, 1, 0, 0, 0, 216, 1058, 1, 0, 0, 0, 218, 1070, 1, 0, 0, 0, 220, 1073, 1, 0, 0, 0, 222, 1077, 1, 0, 0, 0, 224, 1081, 1, 0, 0, 0, 226, 1085, 1, 0, 0, 0, 228, 1089, 1, 0, 0, 0, 230, 1093, 1, 0, 0, 0, 232, 1097, 1, 0, 0, 0, 234, 1102, 1, 0, 0, 0, 236, 1106, 1, 0, 0, 0, 238, 1110, 1, 0, 0, 0, 240, 1114, 1, 0, 0, 0, 242, 1122, 1, 0, 0, 0, 244, 1143, 1, 0, 0, 0, 246, 1147, 1, 0, 0, 0, 248, 1151, 1, 0, 0, 0, 250, 1155, 1, 0, 0, 0, 252, 1159, 1, 0, 0, 0, 254, 1163, 1, 0, 0, 0, 256, 1168, 1, 0, 0, 0, 258, 1172, 1, 0, 0, 0, 260, 1176, 1, 0, 0, 0, 262, 1180, 1, 0, 0, 0, 264, 1184, 1, 0, 0, 0, 266, 1188, 1, 0, 0, 0, 268, 1191, 1, 0, 0, 0, 270, 1195, 1, 0, 0, 0, 272, 1199, 1, 0, 0, 0, 274, 1203, 1, 0, 0, 0, 276, 1207, 1, 0, 0, 0, 278, 1212, 1, 0, 0, 0, 280, 1217, 1, 0, 0, 0, 282, 1222, 1, 0, 0, 0, 284, 1229, 1, 0, 0, 0, 286, 1238, 1, 0, 0, 0, 288, 1245, 1, 0, 0, 0, 290, 1249, 1, 0, 0, 0, 292, 1253, 1, 0, 0, 0, 294, 1257, 1, 0, 0, 0, 296, 1261, 1, 0, 0, 0, 298, 1267, 1, 0, 0, 0, 300, 1271, 1, 0, 0, 0, 302, 1275, 1, 0, 0, 0, 304, 1279, 1, 0, 0, 0, 306, 1283, 1, 0, 0, 0, 308, 1287, 1, 0, 0, 0, 310, 1291, 1, 0, 0, 0, 312, 1295, 1, 0, 0, 0, 314, 1299, 1, 0, 0, 0, 316, 1303, 1, 0, 0, 0, 318, 1307, 1, 0, 0, 0, 320, 1311, 1, 0, 0, 0, 322, 1316, 1, 0, 0, 0, 324, 1320, 1, 0, 0, 0, 326, 1324, 1, 0, 0, 0, 328, 1328, 1, 0, 0, 0, 330, 1332, 1, 0, 0, 0, 332, 1336, 1, 0, 0, 0, 334, 1340, 1, 0, 0, 0, 336, 1344, 1, 0, 0, 0, 338, 1348, 1, 0, 0, 0, 340, 1353, 1, 0, 0, 0, 342, 1358, 1, 0, 0, 0, 344, 1362, 1, 0, 0, 0, 346, 1366, 1, 0, 0, 0, 348, 1370, 1, 0, 0, 0, 350, 1375, 1, 0, 0, 0, 352, 1384, 1, 0, 0, 0, 354, 1388, 1, 0, 0, 0, 356, 1392, 1, 0, 0, 0, 358, 1396, 1, 0, 0, 0, 360, 1400, 1, 0, 0, 0, 362, 1405, 1, 0, 0, 0, 364, 1409, 1, 0, 0, 0, 366, 1413, 1, 0, 0, 0, 368, 1417, 1, 0, 0, 0, 370, 1422, 1, 0, 0, 0, 372, 1426, 1, 0, 0, 0, 374, 1430, 1, 0, 0, 0, 376, 1434, 1, 0, 0, 0, 378, 1438, 1, 0, 0, 0, 380, 1442, 1, 0, 0, 0, 382, 1448, 1, 0, 0, 0, 384, 1452, 1, 0, 0, 0, 386, 1456, 1, 0, 0, 0, 388, 1460, 1, 0, 0, 0, 390, 1464, 1, 0, 0, 0, 392, 1468, 1, 0, 0, 0, 394, 1472, 1, 0, 0, 0, 396, 1477, 1, 0, 0, 0, 398, 1482, 1, 0, 0, 0, 400, 1486, 1, 0, 0, 0, 402, 1492, 1, 0, 0, 0, 404, 1501, 1, 0, 0, 0, 406, 1505, 1, 0, 0, 0, 408, 1509, 1, 0, 0, 0, 410, 1513, 1, 0, 0, 0, 412, 1517, 1, 0, 0, 0, 414, 1521, 1, 0, 0, 0, 416, 1525, 1, 0, 0, 0, 418, 1529, 1, 0, 0, 0, 420, 1533, 1, 0, 0, 0, 422, 1538, 1, 0, 0, 0, 424, 1544, 1, 0, 0, 0, 426, 1550, 1, 0, 0, 0, 428, 1554, 1, 0, 0, 0, 430, 1558, 1, 0, 0, 0, 432, 1562, 1, 0, 0, 0, 434, 1568, 1, 0, 0, 0, 436, 1574, 1, 0, 0, 0, 438, 1578, 1, 0, 0, 0, 440, 1582, 1, 0, 0, 0, 442, 1586, 1, 0, 0, 0, 444, 1592, 1, 0, 0, 0, 446, 1598, 1, 0, 0, 0, 448, 1604, 1, 0, 0, 0, 450, 451, 7, 0, 0, 0, 451, 452, 7, 1, 0, 0, 452, 453, 7, 2, 0, 0, 453, 454, 7, 2, 0, 0, 454, 455, 7, 3, 0, 0, 455, 456, 7, 4, 0, 0, 456, 457, 7, 5, 0, 0, 457, 458, 1, 0, 0, 0, 458, 459, 6, 0, 0, 0, 459, 17, 1, 0, 0, 0, 460, 461, 7, 0, 0, 0, 461, 462, 7, 6, 0, 0, 462, 463, 7, 7, 0, 0, 463, 464, 7, 8, 0, 0, 464, 465, 1, 0, 0, 0, 465, 466, 6, 1, 1, 0, 466, 19, 1, 0, 0, 0, 467, 468, 7, 3, 0, 0, 468, 469, 7, 9, 0, 0, 469, 470, 7, 6, 0, 0, 470, 471, 7, 1, 0, 0, 471, 472, 7, 4, 0, 0, 472, 473, 7, 10, 0, 0, 473, 474, 1, 0, 0, 0, 474, 475, 6, 2, 2, 0, 475, 21, 1, 0, 0, 0, 476, 477, 7, 3, 0, 0, 477, 478, 7, 11, 0, 0, 478, 479, 7, 12, 0, 0, 479, 480, 7, 13, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 3, 0, 0, 482, 23, 1, 0, 0, 0, 483, 484, 7, 3, 0, 0, 484, 485, 7, 14, 0, 0, 485, 486, 7, 8, 0, 0, 486, 487, 7, 13, 0, 0, 487, 488, 7, 12, 0, 0, 488, 489, 7, 1, 0, 0, 489, 490, 7, 9, 0, 0, 490, 491, 1, 0, 0, 0, 491, 492, 6, 4, 3, 0, 492, 25, 1, 0, 0, 0, 493, 494, 7, 15, 0, 0, 494, 495, 7, 6, 0, 0, 495, 496, 7, 7, 0, 0, 496, 497, 7, 16, 0, 0, 497, 498, 1, 0, 0, 0, 498, 499, 6, 5, 4, 0, 499, 27, 1, 0, 0, 0, 500, 501, 7, 17, 0, 0, 501, 502, 7, 6, 0, 0, 502, 503, 7, 7, 0, 0, 503, 504, 7, 18, 0, 0, 504, 505, 1, 0, 0, 0, 505, 506, 6, 6, 0, 0, 506, 29, 1, 0, 0, 0, 507, 508, 7, 18, 0, 0, 508, 509, 7, 3, 0, 0, 509, 510, 7, 3, 0, 0, 510, 511, 7, 8, 0, 0, 511, 512, 1, 0, 0, 0, 512, 513, 6, 7, 1, 0, 513, 31, 1, 0, 0, 0, 514, 515, 7, 13, 0, 0, 515, 516, 7, 1, 0, 0, 516, 517, 7, 16, 0, 0, 517, 518, 7, 1, 0, 0, 518, 519, 7, 5, 0, 0, 519, 520, 1, 0, 0, 0, 520, 521, 6, 8, 0, 0, 521, 33, 1, 0, 0, 0, 522, 523, 7, 16, 0, 0, 523, 524, 7, 11, 0, 0, 524, 525, 5, 95, 0, 0, 525, 526, 7, 3, 0, 0, 526, 527, 7, 14, 0, 0, 527, 528, 7, 8, 0, 0, 528, 529, 7, 12, 0, 0, 529, 530, 7, 9, 0, 0, 530, 531, 7, 0, 0, 0, 531, 532, 1, 0, 0, 0, 532, 533, 6, 9, 5, 0, 533, 35, 1, 0, 0, 0, 534, 535, 7, 6, 0, 0, 535, 536, 7, 3, 0, 0, 536, 537, 7, 9, 0, 0, 537, 538, 7, 12, 0, 0, 538, 539, 7, 16, 0, 0, 539, 540, 7, 3, 0, 0, 540, 541, 1, 0, 0, 0, 541, 542, 6, 10, 6, 0, 542, 37, 1, 0, 0, 0, 543, 544, 7, 6, 0, 0, 544, 545, 7, 7, 0, 0, 545, 546, 7, 19, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 6, 11, 0, 0, 548, 39, 1, 0, 0, 0, 549, 550, 7, 2, 0, 0, 550, 551, 7, 10, 0, 0, 551, 552, 7, 7, 0, 0, 552, 553, 7, 19, 0, 0, 553, 554, 1, 0, 0, 0, 554, 555, 6, 12, 7, 0, 555, 41, 1, 0, 0, 0, 556, 557, 7, 2, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 6, 0, 0, 559, 560, 7, 5, 0, 0, 560, 561, 1, 0, 0, 0, 561, 562, 6, 13, 0, 0, 562, 43, 1, 0, 0, 0, 563, 564, 7, 2, 0, 0, 564, 565, 7, 5, 0, 0, 565, 566, 7, 12, 0, 0, 566, 567, 7, 5, 0, 0, 567, 568, 7, 2, 0, 0, 568, 569, 1, 0, 0, 0, 569, 570, 6, 14, 0, 0, 570, 45, 1, 0, 0, 0, 571, 572, 7, 19, 0, 0, 572, 573, 7, 10, 0, 0, 573, 574, 7, 3, 0, 0, 574, 575, 7, 6, 0, 0, 575, 576, 7, 3, 0, 0, 576, 577, 1, 0, 0, 0, 577, 578, 6, 15, 0, 0, 578, 47, 1, 0, 0, 0, 579, 580, 7, 13, 0, 0, 580, 581, 7, 7, 0, 0, 581, 582, 7, 7, 0, 0, 582, 583, 7, 18, 0, 0, 583, 584, 7, 20, 0, 0, 584, 585, 7, 8, 0, 0, 585, 586, 1, 0, 0, 0, 586, 587, 6, 16, 8, 0, 587, 49, 1, 0, 0, 0, 588, 589, 4, 17, 0, 0, 589, 590, 7, 1, 0, 0, 590, 591, 7, 9, 0, 0, 591, 592, 7, 13, 0, 0, 592, 593, 7, 1, 0, 0, 593, 594, 7, 9, 0, 0, 594, 595, 7, 3, 0, 0, 595, 596, 7, 2, 0, 0, 596, 597, 7, 5, 0, 0, 597, 598, 7, 12, 0, 0, 598, 599, 7, 5, 0, 0, 599, 600, 7, 2, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 6, 17, 0, 0, 602, 51, 1, 0, 0, 0, 603, 604, 4, 18, 1, 0, 604, 605, 7, 13, 0, 0, 605, 606, 7, 7, 0, 0, 606, 607, 7, 7, 0, 0, 607, 608, 7, 18, 0, 0, 608, 609, 7, 20, 0, 0, 609, 610, 7, 8, 0, 0, 610, 611, 5, 95, 0, 0, 611, 612, 5, 128020, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 18, 9, 0, 614, 53, 1, 0, 0, 0, 615, 616, 4, 19, 2, 0, 616, 617, 7, 16, 0, 0, 617, 618, 7, 3, 0, 0, 618, 619, 7, 5, 0, 0, 619, 620, 7, 6, 0, 0, 620, 621, 7, 1, 0, 0, 621, 622, 7, 4, 0, 0, 622, 623, 7, 2, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 6, 19, 10, 0, 625, 55, 1, 0, 0, 0, 626, 627, 4, 20, 3, 0, 627, 628, 7, 15, 0, 0, 628, 629, 7, 20, 0, 0, 629, 630, 7, 13, 0, 0, 630, 631, 7, 13, 0, 0, 631, 632, 1, 0, 0, 0, 632, 633, 6, 20, 8, 0, 633, 57, 1, 0, 0, 0, 634, 635, 4, 21, 4, 0, 635, 636, 7, 13, 0, 0, 636, 637, 7, 3, 0, 0, 637, 638, 7, 15, 0, 0, 638, 639, 7, 5, 0, 0, 639, 640, 1, 0, 0, 0, 640, 641, 6, 21, 8, 0, 641, 59, 1, 0, 0, 0, 642, 643, 4, 22, 5, 0, 643, 644, 7, 6, 0, 0, 644, 645, 7, 1, 0, 0, 645, 646, 7, 17, 0, 0, 646, 647, 7, 10, 0, 0, 647, 648, 7, 5, 0, 0, 648, 649, 1, 0, 0, 0, 649, 650, 6, 22, 8, 0, 650, 61, 1, 0, 0, 0, 651, 653, 8, 21, 0, 0, 652, 651, 1, 0, 0, 0, 653, 654, 1, 0, 0, 0, 654, 652, 1, 0, 0, 0, 654, 655, 1, 0, 0, 0, 655, 656, 1, 0, 0, 0, 656, 657, 6, 23, 0, 0, 657, 63, 1, 0, 0, 0, 658, 659, 5, 47, 0, 0, 659, 660, 5, 47, 0, 0, 660, 664, 1, 0, 0, 0, 661, 663, 8, 22, 0, 0, 662, 661, 1, 0, 0, 0, 663, 666, 1, 0, 0, 0, 664, 662, 1, 0, 0, 0, 664, 665, 1, 0, 0, 0, 665, 668, 1, 0, 0, 0, 666, 664, 1, 0, 0, 0, 667, 669, 5, 13, 0, 0, 668, 667, 1, 0, 0, 0, 668, 669, 1, 0, 0, 0, 669, 671, 1, 0, 0, 0, 670, 672, 5, 10, 0, 0, 671, 670, 1, 0, 0, 0, 671, 672, 1, 0, 0, 0, 672, 673, 1, 0, 0, 0, 673, 674, 6, 24, 11, 0, 674, 65, 1, 0, 0, 0, 675, 676, 5, 47, 0, 0, 676, 677, 5, 42, 0, 0, 677, 682, 1, 0, 0, 0, 678, 681, 3, 66, 25, 0, 679, 681, 9, 0, 0, 0, 680, 678, 1, 0, 0, 0, 680, 679, 1, 0, 0, 0, 681, 684, 1, 0, 0, 0, 682, 683, 1, 0, 0, 0, 682, 680, 1, 0, 0, 0, 683, 685, 1, 0, 0, 0, 684, 682, 1, 0, 0, 0, 685, 686, 5, 42, 0, 0, 686, 687, 5, 47, 0, 0, 687, 688, 1, 0, 0, 0, 688, 689, 6, 25, 11, 0, 689, 67, 1, 0, 0, 0, 690, 692, 7, 23, 0, 0, 691, 690, 1, 0, 0, 0, 692, 693, 1, 0, 0, 0, 693, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 696, 6, 26, 11, 0, 696, 69, 1, 0, 0, 0, 697, 698, 5, 124, 0, 0, 698, 699, 1, 0, 0, 0, 699, 700, 6, 27, 12, 0, 700, 71, 1, 0, 0, 0, 701, 702, 7, 24, 0, 0, 702, 73, 1, 0, 0, 0, 703, 704, 7, 25, 0, 0, 704, 75, 1, 0, 0, 0, 705, 706, 5, 92, 0, 0, 706, 707, 7, 26, 0, 0, 707, 77, 1, 0, 0, 0, 708, 709, 8, 27, 0, 0, 709, 79, 1, 0, 0, 0, 710, 712, 7, 3, 0, 0, 711, 713, 7, 28, 0, 0, 712, 711, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 715, 1, 0, 0, 0, 714, 716, 3, 72, 28, 0, 715, 714, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 81, 1, 0, 0, 0, 719, 720, 5, 64, 0, 0, 720, 83, 1, 0, 0, 0, 721, 722, 5, 96, 0, 0, 722, 85, 1, 0, 0, 0, 723, 727, 8, 29, 0, 0, 724, 725, 5, 96, 0, 0, 725, 727, 5, 96, 0, 0, 726, 723, 1, 0, 0, 0, 726, 724, 1, 0, 0, 0, 727, 87, 1, 0, 0, 0, 728, 729, 5, 95, 0, 0, 729, 89, 1, 0, 0, 0, 730, 734, 3, 74, 29, 0, 731, 734, 3, 72, 28, 0, 732, 734, 3, 88, 36, 0, 733, 730, 1, 0, 0, 0, 733, 731, 1, 0, 0, 0, 733, 732, 1, 0, 0, 0, 734, 91, 1, 0, 0, 0, 735, 740, 5, 34, 0, 0, 736, 739, 3, 76, 30, 0, 737, 739, 3, 78, 31, 0, 738, 736, 1, 0, 0, 0, 738, 737, 1, 0, 0, 0, 739, 742, 1, 0, 0, 0, 740, 738, 1, 0, 0, 0, 740, 741, 1, 0, 0, 0, 741, 743, 1, 0, 0, 0, 742, 740, 1, 0, 0, 0, 743, 765, 5, 34, 0, 0, 744, 745, 5, 34, 0, 0, 745, 746, 5, 34, 0, 0, 746, 747, 5, 34, 0, 0, 747, 751, 1, 0, 0, 0, 748, 750, 8, 22, 0, 0, 749, 748, 1, 0, 0, 0, 750, 753, 1, 0, 0, 0, 751, 752, 1, 0, 0, 0, 751, 749, 1, 0, 0, 0, 752, 754, 1, 0, 0, 0, 753, 751, 1, 0, 0, 0, 754, 755, 5, 34, 0, 0, 755, 756, 5, 34, 0, 0, 756, 757, 5, 34, 0, 0, 757, 759, 1, 0, 0, 0, 758, 760, 5, 34, 0, 0, 759, 758, 1, 0, 0, 0, 759, 760, 1, 0, 0, 0, 760, 762, 1, 0, 0, 0, 761, 763, 5, 34, 0, 0, 762, 761, 1, 0, 0, 0, 762, 763, 1, 0, 0, 0, 763, 765, 1, 0, 0, 0, 764, 735, 1, 0, 0, 0, 764, 744, 1, 0, 0, 0, 765, 93, 1, 0, 0, 0, 766, 768, 3, 72, 28, 0, 767, 766, 1, 0, 0, 0, 768, 769, 1, 0, 0, 0, 769, 767, 1, 0, 0, 0, 769, 770, 1, 0, 0, 0, 770, 95, 1, 0, 0, 0, 771, 773, 3, 72, 28, 0, 772, 771, 1, 0, 0, 0, 773, 774, 1, 0, 0, 0, 774, 772, 1, 0, 0, 0, 774, 775, 1, 0, 0, 0, 775, 776, 1, 0, 0, 0, 776, 780, 3, 114, 49, 0, 777, 779, 3, 72, 28, 0, 778, 777, 1, 0, 0, 0, 779, 782, 1, 0, 0, 0, 780, 778, 1, 0, 0, 0, 780, 781, 1, 0, 0, 0, 781, 814, 1, 0, 0, 0, 782, 780, 1, 0, 0, 0, 783, 785, 3, 114, 49, 0, 784, 786, 3, 72, 28, 0, 785, 784, 1, 0, 0, 0, 786, 787, 1, 0, 0, 0, 787, 785, 1, 0, 0, 0, 787, 788, 1, 0, 0, 0, 788, 814, 1, 0, 0, 0, 789, 791, 3, 72, 28, 0, 790, 789, 1, 0, 0, 0, 791, 792, 1, 0, 0, 0, 792, 790, 1, 0, 0, 0, 792, 793, 1, 0, 0, 0, 793, 801, 1, 0, 0, 0, 794, 798, 3, 114, 49, 0, 795, 797, 3, 72, 28, 0, 796, 795, 1, 0, 0, 0, 797, 800, 1, 0, 0, 0, 798, 796, 1, 0, 0, 0, 798, 799, 1, 0, 0, 0, 799, 802, 1, 0, 0, 0, 800, 798, 1, 0, 0, 0, 801, 794, 1, 0, 0, 0, 801, 802, 1, 0, 0, 0, 802, 803, 1, 0, 0, 0, 803, 804, 3, 80, 32, 0, 804, 814, 1, 0, 0, 0, 805, 807, 3, 114, 49, 0, 806, 808, 3, 72, 28, 0, 807, 806, 1, 0, 0, 0, 808, 809, 1, 0, 0, 0, 809, 807, 1, 0, 0, 0, 809, 810, 1, 0, 0, 0, 810, 811, 1, 0, 0, 0, 811, 812, 3, 80, 32, 0, 812, 814, 1, 0, 0, 0, 813, 772, 1, 0, 0, 0, 813, 783, 1, 0, 0, 0, 813, 790, 1, 0, 0, 0, 813, 805, 1, 0, 0, 0, 814, 97, 1, 0, 0, 0, 815, 816, 7, 30, 0, 0, 816, 817, 7, 31, 0, 0, 817, 99, 1, 0, 0, 0, 818, 819, 7, 12, 0, 0, 819, 820, 7, 9, 0, 0, 820, 821, 7, 0, 0, 0, 821, 101, 1, 0, 0, 0, 822, 823, 7, 12, 0, 0, 823, 824, 7, 2, 0, 0, 824, 825, 7, 4, 0, 0, 825, 103, 1, 0, 0, 0, 826, 827, 5, 61, 0, 0, 827, 105, 1, 0, 0, 0, 828, 829, 5, 58, 0, 0, 829, 830, 5, 58, 0, 0, 830, 107, 1, 0, 0, 0, 831, 832, 5, 58, 0, 0, 832, 109, 1, 0, 0, 0, 833, 834, 5, 44, 0, 0, 834, 111, 1, 0, 0, 0, 835, 836, 7, 0, 0, 0, 836, 837, 7, 3, 0, 0, 837, 838, 7, 2, 0, 0, 838, 839, 7, 4, 0, 0, 839, 113, 1, 0, 0, 0, 840, 841, 5, 46, 0, 0, 841, 115, 1, 0, 0, 0, 842, 843, 7, 15, 0, 0, 843, 844, 7, 12, 0, 0, 844, 845, 7, 13, 0, 0, 845, 846, 7, 2, 0, 0, 846, 847, 7, 3, 0, 0, 847, 117, 1, 0, 0, 0, 848, 849, 7, 15, 0, 0, 849, 850, 7, 1, 0, 0, 850, 851, 7, 6, 0, 0, 851, 852, 7, 2, 0, 0, 852, 853, 7, 5, 0, 0, 853, 119, 1, 0, 0, 0, 854, 855, 7, 1, 0, 0, 855, 856, 7, 9, 0, 0, 856, 121, 1, 0, 0, 0, 857, 858, 7, 1, 0, 0, 858, 859, 7, 2, 0, 0, 859, 123, 1, 0, 0, 0, 860, 861, 7, 13, 0, 0, 861, 862, 7, 12, 0, 0, 862, 863, 7, 2, 0, 0, 863, 864, 7, 5, 0, 0, 864, 125, 1, 0, 0, 0, 865, 866, 7, 13, 0, 0, 866, 867, 7, 1, 0, 0, 867, 868, 7, 18, 0, 0, 868, 869, 7, 3, 0, 0, 869, 127, 1, 0, 0, 0, 870, 871, 5, 40, 0, 0, 871, 129, 1, 0, 0, 0, 872, 873, 7, 9, 0, 0, 873, 874, 7, 7, 0, 0, 874, 875, 7, 5, 0, 0, 875, 131, 1, 0, 0, 0, 876, 877, 7, 9, 0, 0, 877, 878, 7, 20, 0, 0, 878, 879, 7, 13, 0, 0, 879, 880, 7, 13, 0, 0, 880, 133, 1, 0, 0, 0, 881, 882, 7, 9, 0, 0, 882, 883, 7, 20, 0, 0, 883, 884, 7, 13, 0, 0, 884, 885, 7, 13, 0, 0, 885, 886, 7, 2, 0, 0, 886, 135, 1, 0, 0, 0, 887, 888, 7, 7, 0, 0, 888, 889, 7, 6, 0, 0, 889, 137, 1, 0, 0, 0, 890, 891, 5, 63, 0, 0, 891, 139, 1, 0, 0, 0, 892, 893, 7, 6, 0, 0, 893, 894, 7, 13, 0, 0, 894, 895, 7, 1, 0, 0, 895, 896, 7, 18, 0, 0, 896, 897, 7, 3, 0, 0, 897, 141, 1, 0, 0, 0, 898, 899, 5, 41, 0, 0, 899, 143, 1, 0, 0, 0, 900, 901, 7, 5, 0, 0, 901, 902, 7, 6, 0, 0, 902, 903, 7, 20, 0, 0, 903, 904, 7, 3, 0, 0, 904, 145, 1, 0, 0, 0, 905, 906, 5, 61, 0, 0, 906, 907, 5, 61, 0, 0, 907, 147, 1, 0, 0, 0, 908, 909, 5, 61, 0, 0, 909, 910, 5, 126, 0, 0, 910, 149, 1, 0, 0, 0, 911, 912, 5, 33, 0, 0, 912, 913, 5, 61, 0, 0, 913, 151, 1, 0, 0, 0, 914, 915, 5, 60, 0, 0, 915, 153, 1, 0, 0, 0, 916, 917, 5, 60, 0, 0, 917, 918, 5, 61, 0, 0, 918, 155, 1, 0, 0, 0, 919, 920, 5, 62, 0, 0, 920, 157, 1, 0, 0, 0, 921, 922, 5, 62, 0, 0, 922, 923, 5, 61, 0, 0, 923, 159, 1, 0, 0, 0, 924, 925, 5, 43, 0, 0, 925, 161, 1, 0, 0, 0, 926, 927, 5, 45, 0, 0, 927, 163, 1, 0, 0, 0, 928, 929, 5, 42, 0, 0, 929, 165, 1, 0, 0, 0, 930, 931, 5, 47, 0, 0, 931, 167, 1, 0, 0, 0, 932, 933, 5, 37, 0, 0, 933, 169, 1, 0, 0, 0, 934, 935, 5, 123, 0, 0, 935, 171, 1, 0, 0, 0, 936, 937, 5, 125, 0, 0, 937, 173, 1, 0, 0, 0, 938, 939, 3, 46, 15, 0, 939, 940, 1, 0, 0, 0, 940, 941, 6, 79, 13, 0, 941, 175, 1, 0, 0, 0, 942, 945, 3, 138, 61, 0, 943, 946, 3, 74, 29, 0, 944, 946, 3, 88, 36, 0, 945, 943, 1, 0, 0, 0, 945, 944, 1, 0, 0, 0, 946, 950, 1, 0, 0, 0, 947, 949, 3, 90, 37, 0, 948, 947, 1, 0, 0, 0, 949, 952, 1, 0, 0, 0, 950, 948, 1, 0, 0, 0, 950, 951, 1, 0, 0, 0, 951, 960, 1, 0, 0, 0, 952, 950, 1, 0, 0, 0, 953, 955, 3, 138, 61, 0, 954, 956, 3, 72, 28, 0, 955, 954, 1, 0, 0, 0, 956, 957, 1, 0, 0, 0, 957, 955, 1, 0, 0, 0, 957, 958, 1, 0, 0, 0, 958, 960, 1, 0, 0, 0, 959, 942, 1, 0, 0, 0, 959, 953, 1, 0, 0, 0, 960, 177, 1, 0, 0, 0, 961, 962, 5, 91, 0, 0, 962, 963, 1, 0, 0, 0, 963, 964, 6, 81, 0, 0, 964, 965, 6, 81, 0, 0, 965, 179, 1, 0, 0, 0, 966, 967, 5, 93, 0, 0, 967, 968, 1, 0, 0, 0, 968, 969, 6, 82, 12, 0, 969, 970, 6, 82, 12, 0, 970, 181, 1, 0, 0, 0, 971, 975, 3, 74, 29, 0, 972, 974, 3, 90, 37, 0, 973, 972, 1, 0, 0, 0, 974, 977, 1, 0, 0, 0, 975, 973, 1, 0, 0, 0, 975, 976, 1, 0, 0, 0, 976, 988, 1, 0, 0, 0, 977, 975, 1, 0, 0, 0, 978, 981, 3, 88, 36, 0, 979, 981, 3, 82, 33, 0, 980, 978, 1, 0, 0, 0, 980, 979, 1, 0, 0, 0, 981, 983, 1, 0, 0, 0, 982, 984, 3, 90, 37, 0, 983, 982, 1, 0, 0, 0, 984, 985, 1, 0, 0, 0, 985, 983, 1, 0, 0, 0, 985, 986, 1, 0, 0, 0, 986, 988, 1, 0, 0, 0, 987, 971, 1, 0, 0, 0, 987, 980, 1, 0, 0, 0, 988, 183, 1, 0, 0, 0, 989, 991, 3, 84, 34, 0, 990, 992, 3, 86, 35, 0, 991, 990, 1, 0, 0, 0, 992, 993, 1, 0, 0, 0, 993, 991, 1, 0, 0, 0, 993, 994, 1, 0, 0, 0, 994, 995, 1, 0, 0, 0, 995, 996, 3, 84, 34, 0, 996, 185, 1, 0, 0, 0, 997, 998, 3, 184, 84, 0, 998, 187, 1, 0, 0, 0, 999, 1000, 3, 64, 24, 0, 1000, 1001, 1, 0, 0, 0, 1001, 1002, 6, 86, 11, 0, 1002, 189, 1, 0, 0, 0, 1003, 1004, 3, 66, 25, 0, 1004, 1005, 1, 0, 0, 0, 1005, 1006, 6, 87, 11, 0, 1006, 191, 1, 0, 0, 0, 1007, 1008, 3, 68, 26, 0, 1008, 1009, 1, 0, 0, 0, 1009, 1010, 6, 88, 11, 0, 1010, 193, 1, 0, 0, 0, 1011, 1012, 3, 178, 81, 0, 1012, 1013, 1, 0, 0, 0, 1013, 1014, 6, 89, 14, 0, 1014, 1015, 6, 89, 15, 0, 1015, 195, 1, 0, 0, 0, 1016, 1017, 3, 70, 27, 0, 1017, 1018, 1, 0, 0, 0, 1018, 1019, 6, 90, 16, 0, 1019, 1020, 6, 90, 12, 0, 1020, 197, 1, 0, 0, 0, 1021, 1022, 3, 68, 26, 0, 1022, 1023, 1, 0, 0, 0, 1023, 1024, 6, 91, 11, 0, 1024, 199, 1, 0, 0, 0, 1025, 1026, 3, 64, 24, 0, 1026, 1027, 1, 0, 0, 0, 1027, 1028, 6, 92, 11, 0, 1028, 201, 1, 0, 0, 0, 1029, 1030, 3, 66, 25, 0, 1030, 1031, 1, 0, 0, 0, 1031, 1032, 6, 93, 11, 0, 1032, 203, 1, 0, 0, 0, 1033, 1034, 3, 70, 27, 0, 1034, 1035, 1, 0, 0, 0, 1035, 1036, 6, 94, 16, 0, 1036, 1037, 6, 94, 12, 0, 1037, 205, 1, 0, 0, 0, 1038, 1039, 3, 178, 81, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1041, 6, 95, 14, 0, 1041, 207, 1, 0, 0, 0, 1042, 1043, 3, 180, 82, 0, 1043, 1044, 1, 0, 0, 0, 1044, 1045, 6, 96, 17, 0, 1045, 209, 1, 0, 0, 0, 1046, 1047, 3, 108, 46, 0, 1047, 1048, 1, 0, 0, 0, 1048, 1049, 6, 97, 18, 0, 1049, 211, 1, 0, 0, 0, 1050, 1051, 3, 110, 47, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1053, 6, 98, 19, 0, 1053, 213, 1, 0, 0, 0, 1054, 1055, 3, 104, 44, 0, 1055, 1056, 1, 0, 0, 0, 1056, 1057, 6, 99, 20, 0, 1057, 215, 1, 0, 0, 0, 1058, 1059, 7, 16, 0, 0, 1059, 1060, 7, 3, 0, 0, 1060, 1061, 7, 5, 0, 0, 1061, 1062, 7, 12, 0, 0, 1062, 1063, 7, 0, 0, 0, 1063, 1064, 7, 12, 0, 0, 1064, 1065, 7, 5, 0, 0, 1065, 1066, 7, 12, 0, 0, 1066, 217, 1, 0, 0, 0, 1067, 1071, 8, 32, 0, 0, 1068, 1069, 5, 47, 0, 0, 1069, 1071, 8, 33, 0, 0, 1070, 1067, 1, 0, 0, 0, 1070, 1068, 1, 0, 0, 0, 1071, 219, 1, 0, 0, 0, 1072, 1074, 3, 218, 101, 0, 1073, 1072, 1, 0, 0, 0, 1074, 1075, 1, 0, 0, 0, 1075, 1073, 1, 0, 0, 0, 1075, 1076, 1, 0, 0, 0, 1076, 221, 1, 0, 0, 0, 1077, 1078, 3, 220, 102, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 103, 21, 0, 1080, 223, 1, 0, 0, 0, 1081, 1082, 3, 92, 38, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 104, 22, 0, 1084, 225, 1, 0, 0, 0, 1085, 1086, 3, 64, 24, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1088, 6, 105, 11, 0, 1088, 227, 1, 0, 0, 0, 1089, 1090, 3, 66, 25, 0, 1090, 1091, 1, 0, 0, 0, 1091, 1092, 6, 106, 11, 0, 1092, 229, 1, 0, 0, 0, 1093, 1094, 3, 68, 26, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 107, 11, 0, 1096, 231, 1, 0, 0, 0, 1097, 1098, 3, 70, 27, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 108, 16, 0, 1100, 1101, 6, 108, 12, 0, 1101, 233, 1, 0, 0, 0, 1102, 1103, 3, 114, 49, 0, 1103, 1104, 1, 0, 0, 0, 1104, 1105, 6, 109, 23, 0, 1105, 235, 1, 0, 0, 0, 1106, 1107, 3, 110, 47, 0, 1107, 1108, 1, 0, 0, 0, 1108, 1109, 6, 110, 19, 0, 1109, 237, 1, 0, 0, 0, 1110, 1111, 3, 138, 61, 0, 1111, 1112, 1, 0, 0, 0, 1112, 1113, 6, 111, 24, 0, 1113, 239, 1, 0, 0, 0, 1114, 1115, 3, 176, 80, 0, 1115, 1116, 1, 0, 0, 0, 1116, 1117, 6, 112, 25, 0, 1117, 241, 1, 0, 0, 0, 1118, 1123, 3, 74, 29, 0, 1119, 1123, 3, 72, 28, 0, 1120, 1123, 3, 88, 36, 0, 1121, 1123, 3, 164, 74, 0, 1122, 1118, 1, 0, 0, 0, 1122, 1119, 1, 0, 0, 0, 1122, 1120, 1, 0, 0, 0, 1122, 1121, 1, 0, 0, 0, 1123, 243, 1, 0, 0, 0, 1124, 1127, 3, 74, 29, 0, 1125, 1127, 3, 164, 74, 0, 1126, 1124, 1, 0, 0, 0, 1126, 1125, 1, 0, 0, 0, 1127, 1131, 1, 0, 0, 0, 1128, 1130, 3, 242, 113, 0, 1129, 1128, 1, 0, 0, 0, 1130, 1133, 1, 0, 0, 0, 1131, 1129, 1, 0, 0, 0, 1131, 1132, 1, 0, 0, 0, 1132, 1144, 1, 0, 0, 0, 1133, 1131, 1, 0, 0, 0, 1134, 1137, 3, 88, 36, 0, 1135, 1137, 3, 82, 33, 0, 1136, 1134, 1, 0, 0, 0, 1136, 1135, 1, 0, 0, 0, 1137, 1139, 1, 0, 0, 0, 1138, 1140, 3, 242, 113, 0, 1139, 1138, 1, 0, 0, 0, 1140, 1141, 1, 0, 0, 0, 1141, 1139, 1, 0, 0, 0, 1141, 1142, 1, 0, 0, 0, 1142, 1144, 1, 0, 0, 0, 1143, 1126, 1, 0, 0, 0, 1143, 1136, 1, 0, 0, 0, 1144, 245, 1, 0, 0, 0, 1145, 1148, 3, 244, 114, 0, 1146, 1148, 3, 184, 84, 0, 1147, 1145, 1, 0, 0, 0, 1147, 1146, 1, 0, 0, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1147, 1, 0, 0, 0, 1149, 1150, 1, 0, 0, 0, 1150, 247, 1, 0, 0, 0, 1151, 1152, 3, 64, 24, 0, 1152, 1153, 1, 0, 0, 0, 1153, 1154, 6, 116, 11, 0, 1154, 249, 1, 0, 0, 0, 1155, 1156, 3, 66, 25, 0, 1156, 1157, 1, 0, 0, 0, 1157, 1158, 6, 117, 11, 0, 1158, 251, 1, 0, 0, 0, 1159, 1160, 3, 68, 26, 0, 1160, 1161, 1, 0, 0, 0, 1161, 1162, 6, 118, 11, 0, 1162, 253, 1, 0, 0, 0, 1163, 1164, 3, 70, 27, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1166, 6, 119, 16, 0, 1166, 1167, 6, 119, 12, 0, 1167, 255, 1, 0, 0, 0, 1168, 1169, 3, 104, 44, 0, 1169, 1170, 1, 0, 0, 0, 1170, 1171, 6, 120, 20, 0, 1171, 257, 1, 0, 0, 0, 1172, 1173, 3, 110, 47, 0, 1173, 1174, 1, 0, 0, 0, 1174, 1175, 6, 121, 19, 0, 1175, 259, 1, 0, 0, 0, 1176, 1177, 3, 114, 49, 0, 1177, 1178, 1, 0, 0, 0, 1178, 1179, 6, 122, 23, 0, 1179, 261, 1, 0, 0, 0, 1180, 1181, 3, 138, 61, 0, 1181, 1182, 1, 0, 0, 0, 1182, 1183, 6, 123, 24, 0, 1183, 263, 1, 0, 0, 0, 1184, 1185, 3, 176, 80, 0, 1185, 1186, 1, 0, 0, 0, 1186, 1187, 6, 124, 25, 0, 1187, 265, 1, 0, 0, 0, 1188, 1189, 7, 12, 0, 0, 1189, 1190, 7, 2, 0, 0, 1190, 267, 1, 0, 0, 0, 1191, 1192, 3, 246, 115, 0, 1192, 1193, 1, 0, 0, 0, 1193, 1194, 6, 126, 26, 0, 1194, 269, 1, 0, 0, 0, 1195, 1196, 3, 64, 24, 0, 1196, 1197, 1, 0, 0, 0, 1197, 1198, 6, 127, 11, 0, 1198, 271, 1, 0, 0, 0, 1199, 1200, 3, 66, 25, 0, 1200, 1201, 1, 0, 0, 0, 1201, 1202, 6, 128, 11, 0, 1202, 273, 1, 0, 0, 0, 1203, 1204, 3, 68, 26, 0, 1204, 1205, 1, 0, 0, 0, 1205, 1206, 6, 129, 11, 0, 1206, 275, 1, 0, 0, 0, 1207, 1208, 3, 70, 27, 0, 1208, 1209, 1, 0, 0, 0, 1209, 1210, 6, 130, 16, 0, 1210, 1211, 6, 130, 12, 0, 1211, 277, 1, 0, 0, 0, 1212, 1213, 3, 178, 81, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 131, 14, 0, 1215, 1216, 6, 131, 27, 0, 1216, 279, 1, 0, 0, 0, 1217, 1218, 7, 7, 0, 0, 1218, 1219, 7, 9, 0, 0, 1219, 1220, 1, 0, 0, 0, 1220, 1221, 6, 132, 28, 0, 1221, 281, 1, 0, 0, 0, 1222, 1223, 7, 19, 0, 0, 1223, 1224, 7, 1, 0, 0, 1224, 1225, 7, 5, 0, 0, 1225, 1226, 7, 10, 0, 0, 1226, 1227, 1, 0, 0, 0, 1227, 1228, 6, 133, 28, 0, 1228, 283, 1, 0, 0, 0, 1229, 1230, 8, 34, 0, 0, 1230, 285, 1, 0, 0, 0, 1231, 1233, 3, 284, 134, 0, 1232, 1231, 1, 0, 0, 0, 1233, 1234, 1, 0, 0, 0, 1234, 1232, 1, 0, 0, 0, 1234, 1235, 1, 0, 0, 0, 1235, 1236, 1, 0, 0, 0, 1236, 1237, 3, 108, 46, 0, 1237, 1239, 1, 0, 0, 0, 1238, 1232, 1, 0, 0, 0, 1238, 1239, 1, 0, 0, 0, 1239, 1241, 1, 0, 0, 0, 1240, 1242, 3, 284, 134, 0, 1241, 1240, 1, 0, 0, 0, 1242, 1243, 1, 0, 0, 0, 1243, 1241, 1, 0, 0, 0, 1243, 1244, 1, 0, 0, 0, 1244, 287, 1, 0, 0, 0, 1245, 1246, 3, 286, 135, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 136, 29, 0, 1248, 289, 1, 0, 0, 0, 1249, 1250, 3, 64, 24, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 137, 11, 0, 1252, 291, 1, 0, 0, 0, 1253, 1254, 3, 66, 25, 0, 1254, 1255, 1, 0, 0, 0, 1255, 1256, 6, 138, 11, 0, 1256, 293, 1, 0, 0, 0, 1257, 1258, 3, 68, 26, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 139, 11, 0, 1260, 295, 1, 0, 0, 0, 1261, 1262, 3, 70, 27, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 140, 16, 0, 1264, 1265, 6, 140, 12, 0, 1265, 1266, 6, 140, 12, 0, 1266, 297, 1, 0, 0, 0, 1267, 1268, 3, 104, 44, 0, 1268, 1269, 1, 0, 0, 0, 1269, 1270, 6, 141, 20, 0, 1270, 299, 1, 0, 0, 0, 1271, 1272, 3, 110, 47, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1274, 6, 142, 19, 0, 1274, 301, 1, 0, 0, 0, 1275, 1276, 3, 114, 49, 0, 1276, 1277, 1, 0, 0, 0, 1277, 1278, 6, 143, 23, 0, 1278, 303, 1, 0, 0, 0, 1279, 1280, 3, 282, 133, 0, 1280, 1281, 1, 0, 0, 0, 1281, 1282, 6, 144, 30, 0, 1282, 305, 1, 0, 0, 0, 1283, 1284, 3, 246, 115, 0, 1284, 1285, 1, 0, 0, 0, 1285, 1286, 6, 145, 26, 0, 1286, 307, 1, 0, 0, 0, 1287, 1288, 3, 186, 85, 0, 1288, 1289, 1, 0, 0, 0, 1289, 1290, 6, 146, 31, 0, 1290, 309, 1, 0, 0, 0, 1291, 1292, 3, 138, 61, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 147, 24, 0, 1294, 311, 1, 0, 0, 0, 1295, 1296, 3, 176, 80, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 148, 25, 0, 1298, 313, 1, 0, 0, 0, 1299, 1300, 3, 64, 24, 0, 1300, 1301, 1, 0, 0, 0, 1301, 1302, 6, 149, 11, 0, 1302, 315, 1, 0, 0, 0, 1303, 1304, 3, 66, 25, 0, 1304, 1305, 1, 0, 0, 0, 1305, 1306, 6, 150, 11, 0, 1306, 317, 1, 0, 0, 0, 1307, 1308, 3, 68, 26, 0, 1308, 1309, 1, 0, 0, 0, 1309, 1310, 6, 151, 11, 0, 1310, 319, 1, 0, 0, 0, 1311, 1312, 3, 70, 27, 0, 1312, 1313, 1, 0, 0, 0, 1313, 1314, 6, 152, 16, 0, 1314, 1315, 6, 152, 12, 0, 1315, 321, 1, 0, 0, 0, 1316, 1317, 3, 114, 49, 0, 1317, 1318, 1, 0, 0, 0, 1318, 1319, 6, 153, 23, 0, 1319, 323, 1, 0, 0, 0, 1320, 1321, 3, 138, 61, 0, 1321, 1322, 1, 0, 0, 0, 1322, 1323, 6, 154, 24, 0, 1323, 325, 1, 0, 0, 0, 1324, 1325, 3, 176, 80, 0, 1325, 1326, 1, 0, 0, 0, 1326, 1327, 6, 155, 25, 0, 1327, 327, 1, 0, 0, 0, 1328, 1329, 3, 186, 85, 0, 1329, 1330, 1, 0, 0, 0, 1330, 1331, 6, 156, 31, 0, 1331, 329, 1, 0, 0, 0, 1332, 1333, 3, 182, 83, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1335, 6, 157, 32, 0, 1335, 331, 1, 0, 0, 0, 1336, 1337, 3, 64, 24, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 158, 11, 0, 1339, 333, 1, 0, 0, 0, 1340, 1341, 3, 66, 25, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 159, 11, 0, 1343, 335, 1, 0, 0, 0, 1344, 1345, 3, 68, 26, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 160, 11, 0, 1347, 337, 1, 0, 0, 0, 1348, 1349, 3, 70, 27, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 161, 16, 0, 1351, 1352, 6, 161, 12, 0, 1352, 339, 1, 0, 0, 0, 1353, 1354, 7, 1, 0, 0, 1354, 1355, 7, 9, 0, 0, 1355, 1356, 7, 15, 0, 0, 1356, 1357, 7, 7, 0, 0, 1357, 341, 1, 0, 0, 0, 1358, 1359, 3, 64, 24, 0, 1359, 1360, 1, 0, 0, 0, 1360, 1361, 6, 163, 11, 0, 1361, 343, 1, 0, 0, 0, 1362, 1363, 3, 66, 25, 0, 1363, 1364, 1, 0, 0, 0, 1364, 1365, 6, 164, 11, 0, 1365, 345, 1, 0, 0, 0, 1366, 1367, 3, 68, 26, 0, 1367, 1368, 1, 0, 0, 0, 1368, 1369, 6, 165, 11, 0, 1369, 347, 1, 0, 0, 0, 1370, 1371, 3, 180, 82, 0, 1371, 1372, 1, 0, 0, 0, 1372, 1373, 6, 166, 17, 0, 1373, 1374, 6, 166, 12, 0, 1374, 349, 1, 0, 0, 0, 1375, 1376, 3, 108, 46, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 167, 18, 0, 1378, 351, 1, 0, 0, 0, 1379, 1385, 3, 82, 33, 0, 1380, 1385, 3, 72, 28, 0, 1381, 1385, 3, 114, 49, 0, 1382, 1385, 3, 74, 29, 0, 1383, 1385, 3, 88, 36, 0, 1384, 1379, 1, 0, 0, 0, 1384, 1380, 1, 0, 0, 0, 1384, 1381, 1, 0, 0, 0, 1384, 1382, 1, 0, 0, 0, 1384, 1383, 1, 0, 0, 0, 1385, 1386, 1, 0, 0, 0, 1386, 1384, 1, 0, 0, 0, 1386, 1387, 1, 0, 0, 0, 1387, 353, 1, 0, 0, 0, 1388, 1389, 3, 64, 24, 0, 1389, 1390, 1, 0, 0, 0, 1390, 1391, 6, 169, 11, 0, 1391, 355, 1, 0, 0, 0, 1392, 1393, 3, 66, 25, 0, 1393, 1394, 1, 0, 0, 0, 1394, 1395, 6, 170, 11, 0, 1395, 357, 1, 0, 0, 0, 1396, 1397, 3, 68, 26, 0, 1397, 1398, 1, 0, 0, 0, 1398, 1399, 6, 171, 11, 0, 1399, 359, 1, 0, 0, 0, 1400, 1401, 3, 70, 27, 0, 1401, 1402, 1, 0, 0, 0, 1402, 1403, 6, 172, 16, 0, 1403, 1404, 6, 172, 12, 0, 1404, 361, 1, 0, 0, 0, 1405, 1406, 3, 108, 46, 0, 1406, 1407, 1, 0, 0, 0, 1407, 1408, 6, 173, 18, 0, 1408, 363, 1, 0, 0, 0, 1409, 1410, 3, 110, 47, 0, 1410, 1411, 1, 0, 0, 0, 1411, 1412, 6, 174, 19, 0, 1412, 365, 1, 0, 0, 0, 1413, 1414, 3, 114, 49, 0, 1414, 1415, 1, 0, 0, 0, 1415, 1416, 6, 175, 23, 0, 1416, 367, 1, 0, 0, 0, 1417, 1418, 3, 280, 132, 0, 1418, 1419, 1, 0, 0, 0, 1419, 1420, 6, 176, 33, 0, 1420, 1421, 6, 176, 34, 0, 1421, 369, 1, 0, 0, 0, 1422, 1423, 3, 220, 102, 0, 1423, 1424, 1, 0, 0, 0, 1424, 1425, 6, 177, 21, 0, 1425, 371, 1, 0, 0, 0, 1426, 1427, 3, 92, 38, 0, 1427, 1428, 1, 0, 0, 0, 1428, 1429, 6, 178, 22, 0, 1429, 373, 1, 0, 0, 0, 1430, 1431, 3, 64, 24, 0, 1431, 1432, 1, 0, 0, 0, 1432, 1433, 6, 179, 11, 0, 1433, 375, 1, 0, 0, 0, 1434, 1435, 3, 66, 25, 0, 1435, 1436, 1, 0, 0, 0, 1436, 1437, 6, 180, 11, 0, 1437, 377, 1, 0, 0, 0, 1438, 1439, 3, 68, 26, 0, 1439, 1440, 1, 0, 0, 0, 1440, 1441, 6, 181, 11, 0, 1441, 379, 1, 0, 0, 0, 1442, 1443, 3, 70, 27, 0, 1443, 1444, 1, 0, 0, 0, 1444, 1445, 6, 182, 16, 0, 1445, 1446, 6, 182, 12, 0, 1446, 1447, 6, 182, 12, 0, 1447, 381, 1, 0, 0, 0, 1448, 1449, 3, 110, 47, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 183, 19, 0, 1451, 383, 1, 0, 0, 0, 1452, 1453, 3, 114, 49, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 184, 23, 0, 1455, 385, 1, 0, 0, 0, 1456, 1457, 3, 246, 115, 0, 1457, 1458, 1, 0, 0, 0, 1458, 1459, 6, 185, 26, 0, 1459, 387, 1, 0, 0, 0, 1460, 1461, 3, 64, 24, 0, 1461, 1462, 1, 0, 0, 0, 1462, 1463, 6, 186, 11, 0, 1463, 389, 1, 0, 0, 0, 1464, 1465, 3, 66, 25, 0, 1465, 1466, 1, 0, 0, 0, 1466, 1467, 6, 187, 11, 0, 1467, 391, 1, 0, 0, 0, 1468, 1469, 3, 68, 26, 0, 1469, 1470, 1, 0, 0, 0, 1470, 1471, 6, 188, 11, 0, 1471, 393, 1, 0, 0, 0, 1472, 1473, 3, 70, 27, 0, 1473, 1474, 1, 0, 0, 0, 1474, 1475, 6, 189, 16, 0, 1475, 1476, 6, 189, 12, 0, 1476, 395, 1, 0, 0, 0, 1477, 1478, 7, 35, 0, 0, 1478, 1479, 7, 7, 0, 0, 1479, 1480, 7, 1, 0, 0, 1480, 1481, 7, 9, 0, 0, 1481, 397, 1, 0, 0, 0, 1482, 1483, 3, 266, 125, 0, 1483, 1484, 1, 0, 0, 0, 1484, 1485, 6, 191, 35, 0, 1485, 399, 1, 0, 0, 0, 1486, 1487, 3, 280, 132, 0, 1487, 1488, 1, 0, 0, 0, 1488, 1489, 6, 192, 33, 0, 1489, 1490, 6, 192, 12, 0, 1490, 1491, 6, 192, 0, 0, 1491, 401, 1, 0, 0, 0, 1492, 1493, 7, 20, 0, 0, 1493, 1494, 7, 2, 0, 0, 1494, 1495, 7, 1, 0, 0, 1495, 1496, 7, 9, 0, 0, 1496, 1497, 7, 17, 0, 0, 1497, 1498, 1, 0, 0, 0, 1498, 1499, 6, 193, 12, 0, 1499, 1500, 6, 193, 0, 0, 1500, 403, 1, 0, 0, 0, 1501, 1502, 3, 220, 102, 0, 1502, 1503, 1, 0, 0, 0, 1503, 1504, 6, 194, 21, 0, 1504, 405, 1, 0, 0, 0, 1505, 1506, 3, 92, 38, 0, 1506, 1507, 1, 0, 0, 0, 1507, 1508, 6, 195, 22, 0, 1508, 407, 1, 0, 0, 0, 1509, 1510, 3, 108, 46, 0, 1510, 1511, 1, 0, 0, 0, 1511, 1512, 6, 196, 18, 0, 1512, 409, 1, 0, 0, 0, 1513, 1514, 3, 182, 83, 0, 1514, 1515, 1, 0, 0, 0, 1515, 1516, 6, 197, 32, 0, 1516, 411, 1, 0, 0, 0, 1517, 1518, 3, 186, 85, 0, 1518, 1519, 1, 0, 0, 0, 1519, 1520, 6, 198, 31, 0, 1520, 413, 1, 0, 0, 0, 1521, 1522, 3, 64, 24, 0, 1522, 1523, 1, 0, 0, 0, 1523, 1524, 6, 199, 11, 0, 1524, 415, 1, 0, 0, 0, 1525, 1526, 3, 66, 25, 0, 1526, 1527, 1, 0, 0, 0, 1527, 1528, 6, 200, 11, 0, 1528, 417, 1, 0, 0, 0, 1529, 1530, 3, 68, 26, 0, 1530, 1531, 1, 0, 0, 0, 1531, 1532, 6, 201, 11, 0, 1532, 419, 1, 0, 0, 0, 1533, 1534, 3, 70, 27, 0, 1534, 1535, 1, 0, 0, 0, 1535, 1536, 6, 202, 16, 0, 1536, 1537, 6, 202, 12, 0, 1537, 421, 1, 0, 0, 0, 1538, 1539, 3, 220, 102, 0, 1539, 1540, 1, 0, 0, 0, 1540, 1541, 6, 203, 21, 0, 1541, 1542, 6, 203, 12, 0, 1542, 1543, 6, 203, 36, 0, 1543, 423, 1, 0, 0, 0, 1544, 1545, 3, 92, 38, 0, 1545, 1546, 1, 0, 0, 0, 1546, 1547, 6, 204, 22, 0, 1547, 1548, 6, 204, 12, 0, 1548, 1549, 6, 204, 36, 0, 1549, 425, 1, 0, 0, 0, 1550, 1551, 3, 64, 24, 0, 1551, 1552, 1, 0, 0, 0, 1552, 1553, 6, 205, 11, 0, 1553, 427, 1, 0, 0, 0, 1554, 1555, 3, 66, 25, 0, 1555, 1556, 1, 0, 0, 0, 1556, 1557, 6, 206, 11, 0, 1557, 429, 1, 0, 0, 0, 1558, 1559, 3, 68, 26, 0, 1559, 1560, 1, 0, 0, 0, 1560, 1561, 6, 207, 11, 0, 1561, 431, 1, 0, 0, 0, 1562, 1563, 3, 108, 46, 0, 1563, 1564, 1, 0, 0, 0, 1564, 1565, 6, 208, 18, 0, 1565, 1566, 6, 208, 12, 0, 1566, 1567, 6, 208, 10, 0, 1567, 433, 1, 0, 0, 0, 1568, 1569, 3, 110, 47, 0, 1569, 1570, 1, 0, 0, 0, 1570, 1571, 6, 209, 19, 0, 1571, 1572, 6, 209, 12, 0, 1572, 1573, 6, 209, 10, 0, 1573, 435, 1, 0, 0, 0, 1574, 1575, 3, 64, 24, 0, 1575, 1576, 1, 0, 0, 0, 1576, 1577, 6, 210, 11, 0, 1577, 437, 1, 0, 0, 0, 1578, 1579, 3, 66, 25, 0, 1579, 1580, 1, 0, 0, 0, 1580, 1581, 6, 211, 11, 0, 1581, 439, 1, 0, 0, 0, 1582, 1583, 3, 68, 26, 0, 1583, 1584, 1, 0, 0, 0, 1584, 1585, 6, 212, 11, 0, 1585, 441, 1, 0, 0, 0, 1586, 1587, 3, 186, 85, 0, 1587, 1588, 1, 0, 0, 0, 1588, 1589, 6, 213, 12, 0, 1589, 1590, 6, 213, 0, 0, 1590, 1591, 6, 213, 31, 0, 1591, 443, 1, 0, 0, 0, 1592, 1593, 3, 182, 83, 0, 1593, 1594, 1, 0, 0, 0, 1594, 1595, 6, 214, 12, 0, 1595, 1596, 6, 214, 0, 0, 1596, 1597, 6, 214, 32, 0, 1597, 445, 1, 0, 0, 0, 1598, 1599, 3, 98, 41, 0, 1599, 1600, 1, 0, 0, 0, 1600, 1601, 6, 215, 12, 0, 1601, 1602, 6, 215, 0, 0, 1602, 1603, 6, 215, 37, 0, 1603, 447, 1, 0, 0, 0, 1604, 1605, 3, 70, 27, 0, 1605, 1606, 1, 0, 0, 0, 1606, 1607, 6, 216, 16, 0, 1607, 1608, 6, 216, 12, 0, 1608, 449, 1, 0, 0, 0, 66, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 654, 664, 668, 671, 680, 682, 693, 712, 717, 726, 733, 738, 740, 751, 759, 762, 764, 769, 774, 780, 787, 792, 798, 801, 809, 813, 945, 950, 957, 959, 975, 980, 985, 987, 993, 1070, 1075, 1122, 1126, 1131, 1136, 1141, 1143, 1147, 1149, 1234, 1238, 1243, 1384, 1386, 38, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 13, 0, 5, 11, 0, 5, 14, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 71, 0, 5, 0, 0, 7, 28, 0, 7, 72, 0, 7, 37, 0, 7, 38, 0, 7, 35, 0, 7, 82, 0, 7, 29, 0, 7, 40, 0, 7, 52, 0, 7, 70, 0, 7, 86, 0, 5, 10, 0, 5, 7, 0, 7, 96, 0, 7, 95, 0, 7, 74, 0, 7, 73, 0, 7, 94, 0, 5, 12, 0, 7, 90, 0, 5, 15, 0, 7, 32, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 28358a0f614e6..956cfe8649d5f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -27,31 +27,31 @@ public class EsqlBaseLexer extends LexerConfig { public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, DEV_JOIN=20, - DEV_JOIN_FULL=21, DEV_JOIN_LEFT=22, DEV_JOIN_RIGHT=23, DEV_JOIN_LOOKUP=24, - UNKNOWN_CMD=25, LINE_COMMENT=26, MULTILINE_COMMENT=27, WS=28, PIPE=29, - QUOTED_STRING=30, INTEGER_LITERAL=31, DECIMAL_LITERAL=32, BY=33, AND=34, - ASC=35, ASSIGN=36, CAST_OP=37, COLON=38, COMMA=39, DESC=40, DOT=41, FALSE=42, - FIRST=43, IN=44, IS=45, LAST=46, LIKE=47, LP=48, NOT=49, NULL=50, NULLS=51, - OR=52, PARAM=53, RLIKE=54, RP=55, TRUE=56, EQ=57, CIEQ=58, NEQ=59, LT=60, - LTE=61, GT=62, GTE=63, PLUS=64, MINUS=65, ASTERISK=66, SLASH=67, PERCENT=68, - LEFT_BRACES=69, RIGHT_BRACES=70, NAMED_OR_POSITIONAL_PARAM=71, OPENING_BRACKET=72, - CLOSING_BRACKET=73, UNQUOTED_IDENTIFIER=74, QUOTED_IDENTIFIER=75, EXPR_LINE_COMMENT=76, - EXPR_MULTILINE_COMMENT=77, EXPR_WS=78, EXPLAIN_WS=79, EXPLAIN_LINE_COMMENT=80, - EXPLAIN_MULTILINE_COMMENT=81, METADATA=82, UNQUOTED_SOURCE=83, FROM_LINE_COMMENT=84, - FROM_MULTILINE_COMMENT=85, FROM_WS=86, ID_PATTERN=87, PROJECT_LINE_COMMENT=88, - PROJECT_MULTILINE_COMMENT=89, PROJECT_WS=90, AS=91, RENAME_LINE_COMMENT=92, - RENAME_MULTILINE_COMMENT=93, RENAME_WS=94, ON=95, WITH=96, ENRICH_POLICY_NAME=97, - ENRICH_LINE_COMMENT=98, ENRICH_MULTILINE_COMMENT=99, ENRICH_WS=100, ENRICH_FIELD_LINE_COMMENT=101, - ENRICH_FIELD_MULTILINE_COMMENT=102, ENRICH_FIELD_WS=103, MVEXPAND_LINE_COMMENT=104, - MVEXPAND_MULTILINE_COMMENT=105, MVEXPAND_WS=106, INFO=107, SHOW_LINE_COMMENT=108, - SHOW_MULTILINE_COMMENT=109, SHOW_WS=110, SETTING=111, SETTING_LINE_COMMENT=112, - SETTTING_MULTILINE_COMMENT=113, SETTING_WS=114, LOOKUP_LINE_COMMENT=115, - LOOKUP_MULTILINE_COMMENT=116, LOOKUP_WS=117, LOOKUP_FIELD_LINE_COMMENT=118, - LOOKUP_FIELD_MULTILINE_COMMENT=119, LOOKUP_FIELD_WS=120, USING=121, JOIN_LINE_COMMENT=122, - JOIN_MULTILINE_COMMENT=123, JOIN_WS=124, METRICS_LINE_COMMENT=125, METRICS_MULTILINE_COMMENT=126, - METRICS_WS=127, CLOSING_METRICS_LINE_COMMENT=128, CLOSING_METRICS_MULTILINE_COMMENT=129, - CLOSING_METRICS_WS=130; + WHERE=16, JOIN_LOOKUP=17, DEV_INLINESTATS=18, DEV_LOOKUP=19, DEV_METRICS=20, + DEV_JOIN_FULL=21, DEV_JOIN_LEFT=22, DEV_JOIN_RIGHT=23, UNKNOWN_CMD=24, + LINE_COMMENT=25, MULTILINE_COMMENT=26, WS=27, PIPE=28, QUOTED_STRING=29, + INTEGER_LITERAL=30, DECIMAL_LITERAL=31, BY=32, AND=33, ASC=34, ASSIGN=35, + CAST_OP=36, COLON=37, COMMA=38, DESC=39, DOT=40, FALSE=41, FIRST=42, IN=43, + IS=44, LAST=45, LIKE=46, LP=47, NOT=48, NULL=49, NULLS=50, OR=51, PARAM=52, + RLIKE=53, RP=54, TRUE=55, EQ=56, CIEQ=57, NEQ=58, LT=59, LTE=60, GT=61, + GTE=62, PLUS=63, MINUS=64, ASTERISK=65, SLASH=66, PERCENT=67, LEFT_BRACES=68, + RIGHT_BRACES=69, NAMED_OR_POSITIONAL_PARAM=70, OPENING_BRACKET=71, CLOSING_BRACKET=72, + UNQUOTED_IDENTIFIER=73, QUOTED_IDENTIFIER=74, EXPR_LINE_COMMENT=75, EXPR_MULTILINE_COMMENT=76, + EXPR_WS=77, EXPLAIN_WS=78, EXPLAIN_LINE_COMMENT=79, EXPLAIN_MULTILINE_COMMENT=80, + METADATA=81, UNQUOTED_SOURCE=82, FROM_LINE_COMMENT=83, FROM_MULTILINE_COMMENT=84, + FROM_WS=85, ID_PATTERN=86, PROJECT_LINE_COMMENT=87, PROJECT_MULTILINE_COMMENT=88, + PROJECT_WS=89, AS=90, RENAME_LINE_COMMENT=91, RENAME_MULTILINE_COMMENT=92, + RENAME_WS=93, ON=94, WITH=95, ENRICH_POLICY_NAME=96, ENRICH_LINE_COMMENT=97, + ENRICH_MULTILINE_COMMENT=98, ENRICH_WS=99, ENRICH_FIELD_LINE_COMMENT=100, + ENRICH_FIELD_MULTILINE_COMMENT=101, ENRICH_FIELD_WS=102, MVEXPAND_LINE_COMMENT=103, + MVEXPAND_MULTILINE_COMMENT=104, MVEXPAND_WS=105, INFO=106, SHOW_LINE_COMMENT=107, + SHOW_MULTILINE_COMMENT=108, SHOW_WS=109, SETTING=110, SETTING_LINE_COMMENT=111, + SETTTING_MULTILINE_COMMENT=112, SETTING_WS=113, LOOKUP_LINE_COMMENT=114, + LOOKUP_MULTILINE_COMMENT=115, LOOKUP_WS=116, LOOKUP_FIELD_LINE_COMMENT=117, + LOOKUP_FIELD_MULTILINE_COMMENT=118, LOOKUP_FIELD_WS=119, JOIN=120, USING=121, + JOIN_LINE_COMMENT=122, JOIN_MULTILINE_COMMENT=123, JOIN_WS=124, METRICS_LINE_COMMENT=125, + METRICS_MULTILINE_COMMENT=126, METRICS_WS=127, CLOSING_METRICS_LINE_COMMENT=128, + CLOSING_METRICS_MULTILINE_COMMENT=129, CLOSING_METRICS_WS=130; public static final int EXPRESSION_MODE=1, EXPLAIN_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, SETTING_MODE=10, @@ -71,24 +71,24 @@ private static String[] makeRuleNames() { return new String[] { "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", - "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "DEV_JOIN", "DEV_JOIN_FULL", - "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "DEV_JOIN_LOOKUP", "UNKNOWN_CMD", - "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", - "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", - "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", - "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", - "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NESTED_WHERE", "NAMED_OR_POSITIONAL_PARAM", - "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", - "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", - "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", - "EXPLAIN_MULTILINE_COMMENT", "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", - "FROM_COLON", "FROM_COMMA", "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", - "UNQUOTED_SOURCE", "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", - "FROM_MULTILINE_COMMENT", "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", - "PROJECT_PARAM", "PROJECT_NAMED_OR_POSITIONAL_PARAM", "UNQUOTED_ID_BODY_WITH_PATTERN", + "JOIN_LOOKUP", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "DEV_JOIN_FULL", + "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", + "WS", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", + "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", "UNDERSCORE", + "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", + "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", "COMMA", "DESC", "DOT", + "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", "NULL", "NULLS", + "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", "NEQ", "LT", "LTE", + "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", + "RIGHT_BRACES", "NESTED_WHERE", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", + "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", + "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", + "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", + "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", "FROM_COLON", + "FROM_COMMA", "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", "UNQUOTED_SOURCE", + "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", + "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", "PROJECT_PARAM", + "PROJECT_NAMED_OR_POSITIONAL_PARAM", "UNQUOTED_ID_BODY_WITH_PATTERN", "UNQUOTED_ID_PATTERN", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "RENAME_PIPE", "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", "RENAME_PARAM", "RENAME_NAMED_OR_POSITIONAL_PARAM", "AS", "RENAME_ID_PATTERN", @@ -108,7 +108,7 @@ private static String[] makeRuleNames() { "LOOKUP_UNQUOTED_SOURCE", "LOOKUP_QUOTED_SOURCE", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_PIPE", "LOOKUP_FIELD_COMMA", "LOOKUP_FIELD_DOT", "LOOKUP_FIELD_ID_PATTERN", "LOOKUP_FIELD_LINE_COMMENT", - "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "JOIN_PIPE", "JOIN_JOIN", + "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "JOIN_PIPE", "JOIN", "JOIN_AS", "JOIN_ON", "USING", "JOIN_UNQUOTED_SOURCE", "JOIN_QUOTED_SOURCE", "JOIN_COLON", "JOIN_UNQUOTED_IDENTIFER", "JOIN_QUOTED_IDENTIFIER", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", "METRICS_PIPE", "METRICS_UNQUOTED_SOURCE", @@ -124,7 +124,7 @@ private static String[] makeLiteralNames() { return new String[] { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", - "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, + "'sort'", "'stats'", "'where'", "'lookup'", null, null, null, null, null, null, null, null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "':'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", @@ -134,7 +134,7 @@ private static String[] makeLiteralNames() { null, null, null, null, null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, null, null, null, null, null, null, null, "'info'", null, null, null, null, null, null, null, null, null, - null, null, null, null, "'USING'" + null, null, null, null, "'join'", "'USING'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -142,13 +142,13 @@ private static String[] makeSymbolicNames() { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "DEV_JOIN", - "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "DEV_JOIN_LOOKUP", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", - "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", - "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "WHERE", "JOIN_LOOKUP", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", + "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", "LINE_COMMENT", + "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", + "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", + "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", + "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", @@ -163,8 +163,8 @@ private static String[] makeSymbolicNames() { "SHOW_WS", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", - "USING", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", "METRICS_LINE_COMMENT", - "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", + "JOIN", "USING", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", + "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS" }; } @@ -230,38 +230,18 @@ public EsqlBaseLexer(CharStream input) { @Override public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { - case 16: - return DEV_INLINESTATS_sempred((RuleContext)_localctx, predIndex); case 17: - return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); + return DEV_INLINESTATS_sempred((RuleContext)_localctx, predIndex); case 18: - return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); + return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); case 19: - return DEV_JOIN_sempred((RuleContext)_localctx, predIndex); + return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); case 20: return DEV_JOIN_FULL_sempred((RuleContext)_localctx, predIndex); case 21: return DEV_JOIN_LEFT_sempred((RuleContext)_localctx, predIndex); case 22: return DEV_JOIN_RIGHT_sempred((RuleContext)_localctx, predIndex); - case 23: - return DEV_JOIN_LOOKUP_sempred((RuleContext)_localctx, predIndex); - case 112: - return PROJECT_PARAM_sempred((RuleContext)_localctx, predIndex); - case 113: - return PROJECT_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); - case 124: - return RENAME_PARAM_sempred((RuleContext)_localctx, predIndex); - case 125: - return RENAME_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); - case 148: - return ENRICH_FIELD_PARAM_sempred((RuleContext)_localctx, predIndex); - case 149: - return ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); - case 155: - return MVEXPAND_PARAM_sempred((RuleContext)_localctx, predIndex); - case 156: - return MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); } return true; } @@ -286,100 +266,30 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { } return true; } - private boolean DEV_JOIN_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 3: - return this.isDevVersion(); - } - return true; - } private boolean DEV_JOIN_FULL_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { - case 4: + case 3: return this.isDevVersion(); } return true; } private boolean DEV_JOIN_LEFT_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { - case 5: + case 4: return this.isDevVersion(); } return true; } private boolean DEV_JOIN_RIGHT_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { - case 6: - return this.isDevVersion(); - } - return true; - } - private boolean DEV_JOIN_LOOKUP_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 7: - return this.isDevVersion(); - } - return true; - } - private boolean PROJECT_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 8: - return this.isDevVersion(); - } - return true; - } - private boolean PROJECT_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 9: - return this.isDevVersion(); - } - return true; - } - private boolean RENAME_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 10: - return this.isDevVersion(); - } - return true; - } - private boolean RENAME_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 11: - return this.isDevVersion(); - } - return true; - } - private boolean ENRICH_FIELD_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 12: - return this.isDevVersion(); - } - return true; - } - private boolean ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 13: - return this.isDevVersion(); - } - return true; - } - private boolean MVEXPAND_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 14: - return this.isDevVersion(); - } - return true; - } - private boolean MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 15: + case 5: return this.isDevVersion(); } return true; } public static final String _serializedATN = - "\u0004\u0000\u0082\u065b\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ + "\u0004\u0000\u0082\u0649\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ @@ -442,207 +352,204 @@ private boolean MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx "\u00ce\u0002\u00cf\u0007\u00cf\u0002\u00d0\u0007\u00d0\u0002\u00d1\u0007"+ "\u00d1\u0002\u00d2\u0007\u00d2\u0002\u00d3\u0007\u00d3\u0002\u00d4\u0007"+ "\u00d4\u0002\u00d5\u0007\u00d5\u0002\u00d6\u0007\u00d6\u0002\u00d7\u0007"+ - "\u00d7\u0002\u00d8\u0007\u00d8\u0002\u00d9\u0007\u00d9\u0001\u0000\u0001"+ + "\u00d7\u0002\u00d8\u0007\u00d8\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ - "\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001"+ - "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ - "\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ - "\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b"+ - "\u0001\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ - "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+ - "\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+ - "\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001"+ - "\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ + "\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+ + "\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+ + "\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001"+ + "\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001"+ "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001"+ - "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001"+ - "\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ + "\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+ + "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001"+ "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+ - "\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"+ - "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ + "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0015\u0001"+ "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001"+ - "\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+ - "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0004\u0018\u0298\b\u0018\u000b"+ - "\u0018\f\u0018\u0299\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001"+ - "\u0019\u0001\u0019\u0005\u0019\u02a2\b\u0019\n\u0019\f\u0019\u02a5\t\u0019"+ - "\u0001\u0019\u0003\u0019\u02a8\b\u0019\u0001\u0019\u0003\u0019\u02ab\b"+ - "\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001"+ - "\u001a\u0001\u001a\u0005\u001a\u02b4\b\u001a\n\u001a\f\u001a\u02b7\t\u001a"+ - "\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b"+ - "\u0004\u001b\u02bf\b\u001b\u000b\u001b\f\u001b\u02c0\u0001\u001b\u0001"+ - "\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001d\u0001"+ - "\u001d\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+ - " \u0001 \u0001!\u0001!\u0003!\u02d4\b!\u0001!\u0004!\u02d7\b!\u000b!\f"+ - "!\u02d8\u0001\"\u0001\"\u0001#\u0001#\u0001$\u0001$\u0001$\u0003$\u02e2"+ - "\b$\u0001%\u0001%\u0001&\u0001&\u0001&\u0003&\u02e9\b&\u0001\'\u0001\'"+ - "\u0001\'\u0005\'\u02ee\b\'\n\'\f\'\u02f1\t\'\u0001\'\u0001\'\u0001\'\u0001"+ - "\'\u0001\'\u0001\'\u0005\'\u02f9\b\'\n\'\f\'\u02fc\t\'\u0001\'\u0001\'"+ - "\u0001\'\u0001\'\u0001\'\u0003\'\u0303\b\'\u0001\'\u0003\'\u0306\b\'\u0003"+ - "\'\u0308\b\'\u0001(\u0004(\u030b\b(\u000b(\f(\u030c\u0001)\u0004)\u0310"+ - "\b)\u000b)\f)\u0311\u0001)\u0001)\u0005)\u0316\b)\n)\f)\u0319\t)\u0001"+ - ")\u0001)\u0004)\u031d\b)\u000b)\f)\u031e\u0001)\u0004)\u0322\b)\u000b"+ - ")\f)\u0323\u0001)\u0001)\u0005)\u0328\b)\n)\f)\u032b\t)\u0003)\u032d\b"+ - ")\u0001)\u0001)\u0001)\u0001)\u0004)\u0333\b)\u000b)\f)\u0334\u0001)\u0001"+ - ")\u0003)\u0339\b)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001"+ - ",\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001"+ - "/\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u00012\u0001"+ - "3\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+ - "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00017\u00017\u0001"+ - "7\u00017\u00017\u00018\u00018\u00018\u00018\u00018\u00019\u00019\u0001"+ - ":\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+ - "<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001>\u0001>\u0001"+ - "?\u0001?\u0001?\u0001?\u0001?\u0001?\u0001@\u0001@\u0001A\u0001A\u0001"+ - "A\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001"+ - "D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001F\u0001G\u0001G\u0001H\u0001"+ - "H\u0001H\u0001I\u0001I\u0001J\u0001J\u0001K\u0001K\u0001L\u0001L\u0001"+ - "M\u0001M\u0001N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001"+ - "Q\u0001Q\u0001Q\u0003Q\u03bd\bQ\u0001Q\u0005Q\u03c0\bQ\nQ\fQ\u03c3\tQ"+ - "\u0001Q\u0001Q\u0004Q\u03c7\bQ\u000bQ\fQ\u03c8\u0003Q\u03cb\bQ\u0001R"+ - "\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001"+ - "T\u0001T\u0005T\u03d9\bT\nT\fT\u03dc\tT\u0001T\u0001T\u0003T\u03e0\bT"+ - "\u0001T\u0004T\u03e3\bT\u000bT\fT\u03e4\u0003T\u03e7\bT\u0001U\u0001U"+ - "\u0004U\u03eb\bU\u000bU\fU\u03ec\u0001U\u0001U\u0001V\u0001V\u0001W\u0001"+ - "W\u0001W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001"+ - "Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001"+ - "[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001"+ - "^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001"+ - "`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001"+ - "c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001"+ - "e\u0001e\u0001e\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0003"+ - "f\u043a\bf\u0001g\u0004g\u043d\bg\u000bg\fg\u043e\u0001h\u0001h\u0001"+ - "h\u0001h\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001"+ - "k\u0001k\u0001k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001"+ - "m\u0001m\u0001m\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001"+ - "o\u0001p\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001"+ - "q\u0001r\u0001r\u0001r\u0001r\u0003r\u0470\br\u0001s\u0001s\u0003s\u0474"+ - "\bs\u0001s\u0005s\u0477\bs\ns\fs\u047a\ts\u0001s\u0001s\u0003s\u047e\b"+ - "s\u0001s\u0004s\u0481\bs\u000bs\fs\u0482\u0003s\u0485\bs\u0001t\u0001"+ - "t\u0004t\u0489\bt\u000bt\ft\u048a\u0001u\u0001u\u0001u\u0001u\u0001v\u0001"+ - "v\u0001v\u0001v\u0001w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001"+ - "x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001"+ - "{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001"+ - "}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001"+ - "\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001"+ - "\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0082\u0001\u0082\u0001"+ - "\u0082\u0001\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001"+ - "\u0083\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0084\u0001"+ - "\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0086\u0001"+ - "\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+ - "\u0087\u0001\u0087\u0001\u0088\u0004\u0088\u04e0\b\u0088\u000b\u0088\f"+ - "\u0088\u04e1\u0001\u0088\u0001\u0088\u0003\u0088\u04e6\b\u0088\u0001\u0088"+ - "\u0004\u0088\u04e9\b\u0088\u000b\u0088\f\u0088\u04ea\u0001\u0089\u0001"+ - "\u0089\u0001\u0089\u0001\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001"+ - "\u008a\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001"+ - "\u008c\u0001\u008c\u0001\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001"+ - "\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001\u008e\u0001"+ - "\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001"+ - "\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001"+ - "\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001"+ - "\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001"+ - "\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001"+ - "\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001"+ - "\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001"+ - "\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001"+ - "\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001"+ - "\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+ - "\u009c\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001"+ - "\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001"+ - "\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001"+ - "\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001"+ - "\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001"+ - "\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001"+ - "\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6\u0001"+ - "\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001"+ - "\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001"+ - "\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0004\u00a9\u057c\b\u00a9\u000b"+ - "\u00a9\f\u00a9\u057d\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001"+ - "\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001"+ - "\u00ac\u0001\u00ac\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001"+ - "\u00ad\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001"+ - "\u00af\u0001\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+ - "\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001"+ - "\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001"+ - "\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001"+ - "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001"+ - "\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+ - "\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001"+ - "\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001"+ - "\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001"+ - "\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001"+ - "\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001"+ - "\u00be\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001"+ + "\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+ + "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0004\u0017\u028d"+ + "\b\u0017\u000b\u0017\f\u0017\u028e\u0001\u0017\u0001\u0017\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u0297\b\u0018\n\u0018"+ + "\f\u0018\u029a\t\u0018\u0001\u0018\u0003\u0018\u029d\b\u0018\u0001\u0018"+ + "\u0003\u0018\u02a0\b\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u0019\u0001\u0019\u0005\u0019\u02a9\b\u0019\n\u0019"+ + "\f\u0019\u02ac\t\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u001a\u0004\u001a\u02b4\b\u001a\u000b\u001a\f\u001a"+ + "\u02b5\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+ + "\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001\u001e\u0001"+ + "\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001 \u0003 \u02c9"+ + "\b \u0001 \u0004 \u02cc\b \u000b \f \u02cd\u0001!\u0001!\u0001\"\u0001"+ + "\"\u0001#\u0001#\u0001#\u0003#\u02d7\b#\u0001$\u0001$\u0001%\u0001%\u0001"+ + "%\u0003%\u02de\b%\u0001&\u0001&\u0001&\u0005&\u02e3\b&\n&\f&\u02e6\t&"+ + "\u0001&\u0001&\u0001&\u0001&\u0001&\u0001&\u0005&\u02ee\b&\n&\f&\u02f1"+ + "\t&\u0001&\u0001&\u0001&\u0001&\u0001&\u0003&\u02f8\b&\u0001&\u0003&\u02fb"+ + "\b&\u0003&\u02fd\b&\u0001\'\u0004\'\u0300\b\'\u000b\'\f\'\u0301\u0001"+ + "(\u0004(\u0305\b(\u000b(\f(\u0306\u0001(\u0001(\u0005(\u030b\b(\n(\f("+ + "\u030e\t(\u0001(\u0001(\u0004(\u0312\b(\u000b(\f(\u0313\u0001(\u0004("+ + "\u0317\b(\u000b(\f(\u0318\u0001(\u0001(\u0005(\u031d\b(\n(\f(\u0320\t"+ + "(\u0003(\u0322\b(\u0001(\u0001(\u0001(\u0001(\u0004(\u0328\b(\u000b(\f"+ + "(\u0329\u0001(\u0001(\u0003(\u032e\b(\u0001)\u0001)\u0001)\u0001*\u0001"+ + "*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001"+ + "-\u0001-\u0001.\u0001.\u0001/\u0001/\u00010\u00010\u00010\u00010\u0001"+ + "0\u00011\u00011\u00012\u00012\u00012\u00012\u00012\u00012\u00013\u0001"+ + "3\u00013\u00013\u00013\u00013\u00014\u00014\u00014\u00015\u00015\u0001"+ + "5\u00016\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u0001"+ + "7\u00018\u00018\u00019\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001"+ + ":\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001<\u0001"+ + "<\u0001=\u0001=\u0001>\u0001>\u0001>\u0001>\u0001>\u0001>\u0001?\u0001"+ + "?\u0001@\u0001@\u0001@\u0001@\u0001@\u0001A\u0001A\u0001A\u0001B\u0001"+ + "B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001D\u0001E\u0001E\u0001E\u0001"+ + "F\u0001F\u0001G\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001J\u0001"+ + "J\u0001K\u0001K\u0001L\u0001L\u0001M\u0001M\u0001N\u0001N\u0001O\u0001"+ + "O\u0001O\u0001O\u0001P\u0001P\u0001P\u0003P\u03b2\bP\u0001P\u0005P\u03b5"+ + "\bP\nP\fP\u03b8\tP\u0001P\u0001P\u0004P\u03bc\bP\u000bP\fP\u03bd\u0003"+ + "P\u03c0\bP\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001"+ + "R\u0001R\u0001S\u0001S\u0005S\u03ce\bS\nS\fS\u03d1\tS\u0001S\u0001S\u0003"+ + "S\u03d5\bS\u0001S\u0004S\u03d8\bS\u000bS\fS\u03d9\u0003S\u03dc\bS\u0001"+ + "T\u0001T\u0004T\u03e0\bT\u000bT\fT\u03e1\u0001T\u0001T\u0001U\u0001U\u0001"+ + "V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001"+ + "X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001"+ + "Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001"+ + "]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001"+ + "_\u0001_\u0001_\u0001`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001"+ + "a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001"+ + "d\u0001d\u0001d\u0001d\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001"+ + "e\u0003e\u042f\be\u0001f\u0004f\u0432\bf\u000bf\ff\u0433\u0001g\u0001"+ + "g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001i\u0001"+ + "i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001k\u0001k\u0001l\u0001"+ + "l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001n\u0001n\u0001"+ + "n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001"+ + "q\u0001q\u0001q\u0001q\u0003q\u0463\bq\u0001r\u0001r\u0003r\u0467\br\u0001"+ + "r\u0005r\u046a\br\nr\fr\u046d\tr\u0001r\u0001r\u0003r\u0471\br\u0001r"+ + "\u0004r\u0474\br\u000br\fr\u0475\u0003r\u0478\br\u0001s\u0001s\u0004s"+ + "\u047c\bs\u000bs\fs\u047d\u0001t\u0001t\u0001t\u0001t\u0001u\u0001u\u0001"+ + "u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001w\u0001w\u0001"+ + "w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001z\u0001"+ + "z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001|\u0001"+ + "|\u0001}\u0001}\u0001}\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f"+ + "\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080"+ + "\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0082\u0001\u0082"+ + "\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083\u0001\u0083\u0001\u0083"+ + "\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0084"+ + "\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085"+ + "\u0001\u0085\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0087\u0004\u0087"+ + "\u04d1\b\u0087\u000b\u0087\f\u0087\u04d2\u0001\u0087\u0001\u0087\u0003"+ + "\u0087\u04d7\b\u0087\u0001\u0087\u0004\u0087\u04da\b\u0087\u000b\u0087"+ + "\f\u0087\u04db\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0089"+ + "\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a\u0001\u008a\u0001\u008a"+ + "\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008c"+ + "\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d"+ + "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001\u008e"+ + "\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u0090"+ + "\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091\u0001\u0091"+ + "\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093"+ + "\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094"+ + "\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0096"+ + "\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001\u0097"+ + "\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098"+ + "\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a"+ + "\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009b"+ + "\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009d\u0001\u009d"+ + "\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009e"+ + "\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0"+ + "\u0001\u00a0\u0001\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1"+ + "\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2"+ + "\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4"+ + "\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5"+ + "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7"+ + "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8"+ + "\u0001\u00a8\u0001\u00a8\u0004\u00a8\u0569\b\u00a8\u000b\u00a8\f\u00a8"+ + "\u056a\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001"+ + "\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001"+ + "\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001"+ + "\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae\u0001"+ + "\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af\u0001"+ + "\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b1\u0001"+ + "\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001"+ + "\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001"+ + "\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001"+ + "\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001"+ + "\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001"+ + "\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001"+ + "\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001"+ + "\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001"+ + "\u00bc\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001"+ + "\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf\u0001"+ + "\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001"+ "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001"+ - "\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ - "\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ - "\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4\u0001\u00c4\u0001"+ - "\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001"+ - "\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c7\u0001\u00c7\u0001"+ - "\u00c7\u0001\u00c7\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001"+ - "\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00ca\u0001\u00ca\u0001"+ - "\u00ca\u0001\u00ca\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001"+ + "\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001"+ + "\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3\u0001"+ + "\u00c3\u0001\u00c3\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001"+ + "\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c6\u0001\u00c6\u0001"+ + "\u00c6\u0001\u00c6\u0001\u00c7\u0001\u00c7\u0001\u00c7\u0001\u00c7\u0001"+ + "\u00c8\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001\u00c9\u0001\u00c9\u0001"+ + "\u00c9\u0001\u00c9\u0001\u00ca\u0001\u00ca\u0001\u00ca\u0001\u00ca\u0001"+ + "\u00ca\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001"+ "\u00cb\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001"+ - "\u00cc\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001"+ - "\u00cd\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00cf\u0001"+ - "\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001"+ + "\u00cc\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00ce\u0001"+ + "\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00cf\u0001\u00cf\u0001\u00cf\u0001"+ + "\u00cf\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001"+ "\u00d0\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001"+ - "\u00d1\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001"+ - "\u00d2\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d4\u0001"+ - "\u00d4\u0001\u00d4\u0001\u00d4\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001"+ + "\u00d1\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d3\u0001"+ + "\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d4\u0001\u00d4\u0001\u00d4\u0001"+ + "\u00d4\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001"+ "\u00d5\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001"+ "\u00d6\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001"+ - "\u00d7\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001"+ - "\u00d8\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0002"+ - "\u02b5\u02fa\u0000\u00da\u0010\u0001\u0012\u0002\u0014\u0003\u0016\u0004"+ + "\u00d7\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0002"+ + "\u02aa\u02ef\u0000\u00d9\u0010\u0001\u0012\u0002\u0014\u0003\u0016\u0004"+ "\u0018\u0005\u001a\u0006\u001c\u0007\u001e\b \t\"\n$\u000b&\f(\r*\u000e"+ ",\u000f.\u00100\u00112\u00124\u00136\u00148\u0015:\u0016<\u0017>\u0018"+ - "@\u0019B\u001aD\u001bF\u001cH\u001dJ\u0000L\u0000N\u0000P\u0000R\u0000"+ - "T\u0000V\u0000X\u0000Z\u0000\\\u0000^\u001e`\u001fb d!f\"h#j$l%n&p\'r"+ + "@\u0019B\u001aD\u001bF\u001cH\u0000J\u0000L\u0000N\u0000P\u0000R\u0000"+ + "T\u0000V\u0000X\u0000Z\u0000\\\u001d^\u001e`\u001fb d!f\"h#j$l%n&p\'r"+ "(t)v*x+z,|-~.\u0080/\u00820\u00841\u00862\u00883\u008a4\u008c5\u008e6"+ "\u00907\u00928\u00949\u0096:\u0098;\u009a<\u009c=\u009e>\u00a0?\u00a2"+ - "@\u00a4A\u00a6B\u00a8C\u00aaD\u00acE\u00aeF\u00b0\u0000\u00b2G\u00b4H"+ - "\u00b6I\u00b8J\u00ba\u0000\u00bcK\u00beL\u00c0M\u00c2N\u00c4\u0000\u00c6"+ - "\u0000\u00c8O\u00caP\u00ccQ\u00ce\u0000\u00d0\u0000\u00d2\u0000\u00d4"+ - "\u0000\u00d6\u0000\u00d8\u0000\u00daR\u00dc\u0000\u00deS\u00e0\u0000\u00e2"+ - "\u0000\u00e4T\u00e6U\u00e8V\u00ea\u0000\u00ec\u0000\u00ee\u0000\u00f0"+ - "\u0000\u00f2\u0000\u00f4\u0000\u00f6\u0000\u00f8W\u00faX\u00fcY\u00fe"+ - "Z\u0100\u0000\u0102\u0000\u0104\u0000\u0106\u0000\u0108\u0000\u010a\u0000"+ - "\u010c[\u010e\u0000\u0110\\\u0112]\u0114^\u0116\u0000\u0118\u0000\u011a"+ - "_\u011c`\u011e\u0000\u0120a\u0122\u0000\u0124b\u0126c\u0128d\u012a\u0000"+ - "\u012c\u0000\u012e\u0000\u0130\u0000\u0132\u0000\u0134\u0000\u0136\u0000"+ - "\u0138\u0000\u013a\u0000\u013ce\u013ef\u0140g\u0142\u0000\u0144\u0000"+ - "\u0146\u0000\u0148\u0000\u014a\u0000\u014c\u0000\u014eh\u0150i\u0152j"+ - "\u0154\u0000\u0156k\u0158l\u015am\u015cn\u015e\u0000\u0160\u0000\u0162"+ - "o\u0164p\u0166q\u0168r\u016a\u0000\u016c\u0000\u016e\u0000\u0170\u0000"+ - "\u0172\u0000\u0174\u0000\u0176\u0000\u0178s\u017at\u017cu\u017e\u0000"+ - "\u0180\u0000\u0182\u0000\u0184\u0000\u0186v\u0188w\u018ax\u018c\u0000"+ - "\u018e\u0000\u0190\u0000\u0192\u0000\u0194y\u0196\u0000\u0198\u0000\u019a"+ - "\u0000\u019c\u0000\u019e\u0000\u01a0z\u01a2{\u01a4|\u01a6\u0000\u01a8"+ - "\u0000\u01aa\u0000\u01ac}\u01ae~\u01b0\u007f\u01b2\u0000\u01b4\u0000\u01b6"+ - "\u0080\u01b8\u0081\u01ba\u0082\u01bc\u0000\u01be\u0000\u01c0\u0000\u01c2"+ - "\u0000\u0010\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b"+ - "\f\r\u000e\u000f$\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002"+ - "\u0000EEee\u0002\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000"+ - "OOoo\u0002\u0000PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002"+ - "\u0000AAaa\u0002\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000"+ - "MMmm\u0002\u0000GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0002"+ - "\u0000JJjj\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000"+ - "\t\n\r\r \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004"+ - "\u0000\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002"+ - "\u0000YYyy\u000b\u0000\t\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b"+ - "\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u0676\u0000\u0010\u0001\u0000\u0000"+ + "@\u00a4A\u00a6B\u00a8C\u00aaD\u00acE\u00ae\u0000\u00b0F\u00b2G\u00b4H"+ + "\u00b6I\u00b8\u0000\u00baJ\u00bcK\u00beL\u00c0M\u00c2\u0000\u00c4\u0000"+ + "\u00c6N\u00c8O\u00caP\u00cc\u0000\u00ce\u0000\u00d0\u0000\u00d2\u0000"+ + "\u00d4\u0000\u00d6\u0000\u00d8Q\u00da\u0000\u00dcR\u00de\u0000\u00e0\u0000"+ + "\u00e2S\u00e4T\u00e6U\u00e8\u0000\u00ea\u0000\u00ec\u0000\u00ee\u0000"+ + "\u00f0\u0000\u00f2\u0000\u00f4\u0000\u00f6V\u00f8W\u00faX\u00fcY\u00fe"+ + "\u0000\u0100\u0000\u0102\u0000\u0104\u0000\u0106\u0000\u0108\u0000\u010a"+ + "Z\u010c\u0000\u010e[\u0110\\\u0112]\u0114\u0000\u0116\u0000\u0118^\u011a"+ + "_\u011c\u0000\u011e`\u0120\u0000\u0122a\u0124b\u0126c\u0128\u0000\u012a"+ + "\u0000\u012c\u0000\u012e\u0000\u0130\u0000\u0132\u0000\u0134\u0000\u0136"+ + "\u0000\u0138\u0000\u013ad\u013ce\u013ef\u0140\u0000\u0142\u0000\u0144"+ + "\u0000\u0146\u0000\u0148\u0000\u014a\u0000\u014cg\u014eh\u0150i\u0152"+ + "\u0000\u0154j\u0156k\u0158l\u015am\u015c\u0000\u015e\u0000\u0160n\u0162"+ + "o\u0164p\u0166q\u0168\u0000\u016a\u0000\u016c\u0000\u016e\u0000\u0170"+ + "\u0000\u0172\u0000\u0174\u0000\u0176r\u0178s\u017at\u017c\u0000\u017e"+ + "\u0000\u0180\u0000\u0182\u0000\u0184u\u0186v\u0188w\u018a\u0000\u018c"+ + "x\u018e\u0000\u0190\u0000\u0192y\u0194\u0000\u0196\u0000\u0198\u0000\u019a"+ + "\u0000\u019c\u0000\u019ez\u01a0{\u01a2|\u01a4\u0000\u01a6\u0000\u01a8"+ + "\u0000\u01aa}\u01ac~\u01ae\u007f\u01b0\u0000\u01b2\u0000\u01b4\u0080\u01b6"+ + "\u0081\u01b8\u0082\u01ba\u0000\u01bc\u0000\u01be\u0000\u01c0\u0000\u0010"+ + "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ + "\u000f$\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EE"+ + "ee\u0002\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002"+ + "\u0000PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000"+ + "AAaa\u0002\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002"+ + "\u0000GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000"+ + "\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u0000"+ + "09\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\""+ + "\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b"+ + "\u0000\t\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r"+ + " \"#,,//::<<>?\\\\||\u0002\u0000JJjj\u0664\u0000\u0010\u0001\u0000\u0000"+ "\u0000\u0000\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000"+ "\u0000\u0000\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000"+ "\u0000\u0000\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000"+ @@ -654,792 +561,784 @@ private boolean MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx "4\u0001\u0000\u0000\u0000\u00006\u0001\u0000\u0000\u0000\u00008\u0001"+ "\u0000\u0000\u0000\u0000:\u0001\u0000\u0000\u0000\u0000<\u0001\u0000\u0000"+ "\u0000\u0000>\u0001\u0000\u0000\u0000\u0000@\u0001\u0000\u0000\u0000\u0000"+ - "B\u0001\u0000\u0000\u0000\u0000D\u0001\u0000\u0000\u0000\u0000F\u0001"+ - "\u0000\u0000\u0000\u0001H\u0001\u0000\u0000\u0000\u0001^\u0001\u0000\u0000"+ - "\u0000\u0001`\u0001\u0000\u0000\u0000\u0001b\u0001\u0000\u0000\u0000\u0001"+ - "d\u0001\u0000\u0000\u0000\u0001f\u0001\u0000\u0000\u0000\u0001h\u0001"+ - "\u0000\u0000\u0000\u0001j\u0001\u0000\u0000\u0000\u0001l\u0001\u0000\u0000"+ - "\u0000\u0001n\u0001\u0000\u0000\u0000\u0001p\u0001\u0000\u0000\u0000\u0001"+ - "r\u0001\u0000\u0000\u0000\u0001t\u0001\u0000\u0000\u0000\u0001v\u0001"+ - "\u0000\u0000\u0000\u0001x\u0001\u0000\u0000\u0000\u0001z\u0001\u0000\u0000"+ - "\u0000\u0001|\u0001\u0000\u0000\u0000\u0001~\u0001\u0000\u0000\u0000\u0001"+ - "\u0080\u0001\u0000\u0000\u0000\u0001\u0082\u0001\u0000\u0000\u0000\u0001"+ - "\u0084\u0001\u0000\u0000\u0000\u0001\u0086\u0001\u0000\u0000\u0000\u0001"+ - "\u0088\u0001\u0000\u0000\u0000\u0001\u008a\u0001\u0000\u0000\u0000\u0001"+ - "\u008c\u0001\u0000\u0000\u0000\u0001\u008e\u0001\u0000\u0000\u0000\u0001"+ - "\u0090\u0001\u0000\u0000\u0000\u0001\u0092\u0001\u0000\u0000\u0000\u0001"+ - "\u0094\u0001\u0000\u0000\u0000\u0001\u0096\u0001\u0000\u0000\u0000\u0001"+ - "\u0098\u0001\u0000\u0000\u0000\u0001\u009a\u0001\u0000\u0000\u0000\u0001"+ - "\u009c\u0001\u0000\u0000\u0000\u0001\u009e\u0001\u0000\u0000\u0000\u0001"+ - "\u00a0\u0001\u0000\u0000\u0000\u0001\u00a2\u0001\u0000\u0000\u0000\u0001"+ - "\u00a4\u0001\u0000\u0000\u0000\u0001\u00a6\u0001\u0000\u0000\u0000\u0001"+ - "\u00a8\u0001\u0000\u0000\u0000\u0001\u00aa\u0001\u0000\u0000\u0000\u0001"+ - "\u00ac\u0001\u0000\u0000\u0000\u0001\u00ae\u0001\u0000\u0000\u0000\u0001"+ - "\u00b0\u0001\u0000\u0000\u0000\u0001\u00b2\u0001\u0000\u0000\u0000\u0001"+ - "\u00b4\u0001\u0000\u0000\u0000\u0001\u00b6\u0001\u0000\u0000\u0000\u0001"+ - "\u00b8\u0001\u0000\u0000\u0000\u0001\u00bc\u0001\u0000\u0000\u0000\u0001"+ - "\u00be\u0001\u0000\u0000\u0000\u0001\u00c0\u0001\u0000\u0000\u0000\u0001"+ - "\u00c2\u0001\u0000\u0000\u0000\u0002\u00c4\u0001\u0000\u0000\u0000\u0002"+ - "\u00c6\u0001\u0000\u0000\u0000\u0002\u00c8\u0001\u0000\u0000\u0000\u0002"+ - "\u00ca\u0001\u0000\u0000\u0000\u0002\u00cc\u0001\u0000\u0000\u0000\u0003"+ - "\u00ce\u0001\u0000\u0000\u0000\u0003\u00d0\u0001\u0000\u0000\u0000\u0003"+ - "\u00d2\u0001\u0000\u0000\u0000\u0003\u00d4\u0001\u0000\u0000\u0000\u0003"+ - "\u00d6\u0001\u0000\u0000\u0000\u0003\u00d8\u0001\u0000\u0000\u0000\u0003"+ - "\u00da\u0001\u0000\u0000\u0000\u0003\u00de\u0001\u0000\u0000\u0000\u0003"+ - "\u00e0\u0001\u0000\u0000\u0000\u0003\u00e2\u0001\u0000\u0000\u0000\u0003"+ - "\u00e4\u0001\u0000\u0000\u0000\u0003\u00e6\u0001\u0000\u0000\u0000\u0003"+ - "\u00e8\u0001\u0000\u0000\u0000\u0004\u00ea\u0001\u0000\u0000\u0000\u0004"+ - "\u00ec\u0001\u0000\u0000\u0000\u0004\u00ee\u0001\u0000\u0000\u0000\u0004"+ - "\u00f0\u0001\u0000\u0000\u0000\u0004\u00f2\u0001\u0000\u0000\u0000\u0004"+ - "\u00f8\u0001\u0000\u0000\u0000\u0004\u00fa\u0001\u0000\u0000\u0000\u0004"+ - "\u00fc\u0001\u0000\u0000\u0000\u0004\u00fe\u0001\u0000\u0000\u0000\u0005"+ - "\u0100\u0001\u0000\u0000\u0000\u0005\u0102\u0001\u0000\u0000\u0000\u0005"+ - "\u0104\u0001\u0000\u0000\u0000\u0005\u0106\u0001\u0000\u0000\u0000\u0005"+ - "\u0108\u0001\u0000\u0000\u0000\u0005\u010a\u0001\u0000\u0000\u0000\u0005"+ - "\u010c\u0001\u0000\u0000\u0000\u0005\u010e\u0001\u0000\u0000\u0000\u0005"+ - "\u0110\u0001\u0000\u0000\u0000\u0005\u0112\u0001\u0000\u0000\u0000\u0005"+ - "\u0114\u0001\u0000\u0000\u0000\u0006\u0116\u0001\u0000\u0000\u0000\u0006"+ - "\u0118\u0001\u0000\u0000\u0000\u0006\u011a\u0001\u0000\u0000\u0000\u0006"+ - "\u011c\u0001\u0000\u0000\u0000\u0006\u0120\u0001\u0000\u0000\u0000\u0006"+ - "\u0122\u0001\u0000\u0000\u0000\u0006\u0124\u0001\u0000\u0000\u0000\u0006"+ - "\u0126\u0001\u0000\u0000\u0000\u0006\u0128\u0001\u0000\u0000\u0000\u0007"+ - "\u012a\u0001\u0000\u0000\u0000\u0007\u012c\u0001\u0000\u0000\u0000\u0007"+ - "\u012e\u0001\u0000\u0000\u0000\u0007\u0130\u0001\u0000\u0000\u0000\u0007"+ - "\u0132\u0001\u0000\u0000\u0000\u0007\u0134\u0001\u0000\u0000\u0000\u0007"+ - "\u0136\u0001\u0000\u0000\u0000\u0007\u0138\u0001\u0000\u0000\u0000\u0007"+ - "\u013a\u0001\u0000\u0000\u0000\u0007\u013c\u0001\u0000\u0000\u0000\u0007"+ - "\u013e\u0001\u0000\u0000\u0000\u0007\u0140\u0001\u0000\u0000\u0000\b\u0142"+ - "\u0001\u0000\u0000\u0000\b\u0144\u0001\u0000\u0000\u0000\b\u0146\u0001"+ - "\u0000\u0000\u0000\b\u0148\u0001\u0000\u0000\u0000\b\u014a\u0001\u0000"+ - "\u0000\u0000\b\u014c\u0001\u0000\u0000\u0000\b\u014e\u0001\u0000\u0000"+ - "\u0000\b\u0150\u0001\u0000\u0000\u0000\b\u0152\u0001\u0000\u0000\u0000"+ - "\t\u0154\u0001\u0000\u0000\u0000\t\u0156\u0001\u0000\u0000\u0000\t\u0158"+ - "\u0001\u0000\u0000\u0000\t\u015a\u0001\u0000\u0000\u0000\t\u015c\u0001"+ - "\u0000\u0000\u0000\n\u015e\u0001\u0000\u0000\u0000\n\u0160\u0001\u0000"+ - "\u0000\u0000\n\u0162\u0001\u0000\u0000\u0000\n\u0164\u0001\u0000\u0000"+ - "\u0000\n\u0166\u0001\u0000\u0000\u0000\n\u0168\u0001\u0000\u0000\u0000"+ - "\u000b\u016a\u0001\u0000\u0000\u0000\u000b\u016c\u0001\u0000\u0000\u0000"+ - "\u000b\u016e\u0001\u0000\u0000\u0000\u000b\u0170\u0001\u0000\u0000\u0000"+ - "\u000b\u0172\u0001\u0000\u0000\u0000\u000b\u0174\u0001\u0000\u0000\u0000"+ - "\u000b\u0176\u0001\u0000\u0000\u0000\u000b\u0178\u0001\u0000\u0000\u0000"+ - "\u000b\u017a\u0001\u0000\u0000\u0000\u000b\u017c\u0001\u0000\u0000\u0000"+ + "B\u0001\u0000\u0000\u0000\u0000D\u0001\u0000\u0000\u0000\u0001F\u0001"+ + "\u0000\u0000\u0000\u0001\\\u0001\u0000\u0000\u0000\u0001^\u0001\u0000"+ + "\u0000\u0000\u0001`\u0001\u0000\u0000\u0000\u0001b\u0001\u0000\u0000\u0000"+ + "\u0001d\u0001\u0000\u0000\u0000\u0001f\u0001\u0000\u0000\u0000\u0001h"+ + "\u0001\u0000\u0000\u0000\u0001j\u0001\u0000\u0000\u0000\u0001l\u0001\u0000"+ + "\u0000\u0000\u0001n\u0001\u0000\u0000\u0000\u0001p\u0001\u0000\u0000\u0000"+ + "\u0001r\u0001\u0000\u0000\u0000\u0001t\u0001\u0000\u0000\u0000\u0001v"+ + "\u0001\u0000\u0000\u0000\u0001x\u0001\u0000\u0000\u0000\u0001z\u0001\u0000"+ + "\u0000\u0000\u0001|\u0001\u0000\u0000\u0000\u0001~\u0001\u0000\u0000\u0000"+ + "\u0001\u0080\u0001\u0000\u0000\u0000\u0001\u0082\u0001\u0000\u0000\u0000"+ + "\u0001\u0084\u0001\u0000\u0000\u0000\u0001\u0086\u0001\u0000\u0000\u0000"+ + "\u0001\u0088\u0001\u0000\u0000\u0000\u0001\u008a\u0001\u0000\u0000\u0000"+ + "\u0001\u008c\u0001\u0000\u0000\u0000\u0001\u008e\u0001\u0000\u0000\u0000"+ + "\u0001\u0090\u0001\u0000\u0000\u0000\u0001\u0092\u0001\u0000\u0000\u0000"+ + "\u0001\u0094\u0001\u0000\u0000\u0000\u0001\u0096\u0001\u0000\u0000\u0000"+ + "\u0001\u0098\u0001\u0000\u0000\u0000\u0001\u009a\u0001\u0000\u0000\u0000"+ + "\u0001\u009c\u0001\u0000\u0000\u0000\u0001\u009e\u0001\u0000\u0000\u0000"+ + "\u0001\u00a0\u0001\u0000\u0000\u0000\u0001\u00a2\u0001\u0000\u0000\u0000"+ + "\u0001\u00a4\u0001\u0000\u0000\u0000\u0001\u00a6\u0001\u0000\u0000\u0000"+ + "\u0001\u00a8\u0001\u0000\u0000\u0000\u0001\u00aa\u0001\u0000\u0000\u0000"+ + "\u0001\u00ac\u0001\u0000\u0000\u0000\u0001\u00ae\u0001\u0000\u0000\u0000"+ + "\u0001\u00b0\u0001\u0000\u0000\u0000\u0001\u00b2\u0001\u0000\u0000\u0000"+ + "\u0001\u00b4\u0001\u0000\u0000\u0000\u0001\u00b6\u0001\u0000\u0000\u0000"+ + "\u0001\u00ba\u0001\u0000\u0000\u0000\u0001\u00bc\u0001\u0000\u0000\u0000"+ + "\u0001\u00be\u0001\u0000\u0000\u0000\u0001\u00c0\u0001\u0000\u0000\u0000"+ + "\u0002\u00c2\u0001\u0000\u0000\u0000\u0002\u00c4\u0001\u0000\u0000\u0000"+ + "\u0002\u00c6\u0001\u0000\u0000\u0000\u0002\u00c8\u0001\u0000\u0000\u0000"+ + "\u0002\u00ca\u0001\u0000\u0000\u0000\u0003\u00cc\u0001\u0000\u0000\u0000"+ + "\u0003\u00ce\u0001\u0000\u0000\u0000\u0003\u00d0\u0001\u0000\u0000\u0000"+ + "\u0003\u00d2\u0001\u0000\u0000\u0000\u0003\u00d4\u0001\u0000\u0000\u0000"+ + "\u0003\u00d6\u0001\u0000\u0000\u0000\u0003\u00d8\u0001\u0000\u0000\u0000"+ + "\u0003\u00dc\u0001\u0000\u0000\u0000\u0003\u00de\u0001\u0000\u0000\u0000"+ + "\u0003\u00e0\u0001\u0000\u0000\u0000\u0003\u00e2\u0001\u0000\u0000\u0000"+ + "\u0003\u00e4\u0001\u0000\u0000\u0000\u0003\u00e6\u0001\u0000\u0000\u0000"+ + "\u0004\u00e8\u0001\u0000\u0000\u0000\u0004\u00ea\u0001\u0000\u0000\u0000"+ + "\u0004\u00ec\u0001\u0000\u0000\u0000\u0004\u00ee\u0001\u0000\u0000\u0000"+ + "\u0004\u00f0\u0001\u0000\u0000\u0000\u0004\u00f6\u0001\u0000\u0000\u0000"+ + "\u0004\u00f8\u0001\u0000\u0000\u0000\u0004\u00fa\u0001\u0000\u0000\u0000"+ + "\u0004\u00fc\u0001\u0000\u0000\u0000\u0005\u00fe\u0001\u0000\u0000\u0000"+ + "\u0005\u0100\u0001\u0000\u0000\u0000\u0005\u0102\u0001\u0000\u0000\u0000"+ + "\u0005\u0104\u0001\u0000\u0000\u0000\u0005\u0106\u0001\u0000\u0000\u0000"+ + "\u0005\u0108\u0001\u0000\u0000\u0000\u0005\u010a\u0001\u0000\u0000\u0000"+ + "\u0005\u010c\u0001\u0000\u0000\u0000\u0005\u010e\u0001\u0000\u0000\u0000"+ + "\u0005\u0110\u0001\u0000\u0000\u0000\u0005\u0112\u0001\u0000\u0000\u0000"+ + "\u0006\u0114\u0001\u0000\u0000\u0000\u0006\u0116\u0001\u0000\u0000\u0000"+ + "\u0006\u0118\u0001\u0000\u0000\u0000\u0006\u011a\u0001\u0000\u0000\u0000"+ + "\u0006\u011e\u0001\u0000\u0000\u0000\u0006\u0120\u0001\u0000\u0000\u0000"+ + "\u0006\u0122\u0001\u0000\u0000\u0000\u0006\u0124\u0001\u0000\u0000\u0000"+ + "\u0006\u0126\u0001\u0000\u0000\u0000\u0007\u0128\u0001\u0000\u0000\u0000"+ + "\u0007\u012a\u0001\u0000\u0000\u0000\u0007\u012c\u0001\u0000\u0000\u0000"+ + "\u0007\u012e\u0001\u0000\u0000\u0000\u0007\u0130\u0001\u0000\u0000\u0000"+ + "\u0007\u0132\u0001\u0000\u0000\u0000\u0007\u0134\u0001\u0000\u0000\u0000"+ + "\u0007\u0136\u0001\u0000\u0000\u0000\u0007\u0138\u0001\u0000\u0000\u0000"+ + "\u0007\u013a\u0001\u0000\u0000\u0000\u0007\u013c\u0001\u0000\u0000\u0000"+ + "\u0007\u013e\u0001\u0000\u0000\u0000\b\u0140\u0001\u0000\u0000\u0000\b"+ + "\u0142\u0001\u0000\u0000\u0000\b\u0144\u0001\u0000\u0000\u0000\b\u0146"+ + "\u0001\u0000\u0000\u0000\b\u0148\u0001\u0000\u0000\u0000\b\u014a\u0001"+ + "\u0000\u0000\u0000\b\u014c\u0001\u0000\u0000\u0000\b\u014e\u0001\u0000"+ + "\u0000\u0000\b\u0150\u0001\u0000\u0000\u0000\t\u0152\u0001\u0000\u0000"+ + "\u0000\t\u0154\u0001\u0000\u0000\u0000\t\u0156\u0001\u0000\u0000\u0000"+ + "\t\u0158\u0001\u0000\u0000\u0000\t\u015a\u0001\u0000\u0000\u0000\n\u015c"+ + "\u0001\u0000\u0000\u0000\n\u015e\u0001\u0000\u0000\u0000\n\u0160\u0001"+ + "\u0000\u0000\u0000\n\u0162\u0001\u0000\u0000\u0000\n\u0164\u0001\u0000"+ + "\u0000\u0000\n\u0166\u0001\u0000\u0000\u0000\u000b\u0168\u0001\u0000\u0000"+ + "\u0000\u000b\u016a\u0001\u0000\u0000\u0000\u000b\u016c\u0001\u0000\u0000"+ + "\u0000\u000b\u016e\u0001\u0000\u0000\u0000\u000b\u0170\u0001\u0000\u0000"+ + "\u0000\u000b\u0172\u0001\u0000\u0000\u0000\u000b\u0174\u0001\u0000\u0000"+ + "\u0000\u000b\u0176\u0001\u0000\u0000\u0000\u000b\u0178\u0001\u0000\u0000"+ + "\u0000\u000b\u017a\u0001\u0000\u0000\u0000\f\u017c\u0001\u0000\u0000\u0000"+ "\f\u017e\u0001\u0000\u0000\u0000\f\u0180\u0001\u0000\u0000\u0000\f\u0182"+ "\u0001\u0000\u0000\u0000\f\u0184\u0001\u0000\u0000\u0000\f\u0186\u0001"+ - "\u0000\u0000\u0000\f\u0188\u0001\u0000\u0000\u0000\f\u018a\u0001\u0000"+ + "\u0000\u0000\u0000\f\u0188\u0001\u0000\u0000\u0000\r\u018a\u0001\u0000"+ "\u0000\u0000\r\u018c\u0001\u0000\u0000\u0000\r\u018e\u0001\u0000\u0000"+ "\u0000\r\u0190\u0001\u0000\u0000\u0000\r\u0192\u0001\u0000\u0000\u0000"+ "\r\u0194\u0001\u0000\u0000\u0000\r\u0196\u0001\u0000\u0000\u0000\r\u0198"+ "\u0001\u0000\u0000\u0000\r\u019a\u0001\u0000\u0000\u0000\r\u019c\u0001"+ "\u0000\u0000\u0000\r\u019e\u0001\u0000\u0000\u0000\r\u01a0\u0001\u0000"+ - "\u0000\u0000\r\u01a2\u0001\u0000\u0000\u0000\r\u01a4\u0001\u0000\u0000"+ + "\u0000\u0000\r\u01a2\u0001\u0000\u0000\u0000\u000e\u01a4\u0001\u0000\u0000"+ "\u0000\u000e\u01a6\u0001\u0000\u0000\u0000\u000e\u01a8\u0001\u0000\u0000"+ "\u0000\u000e\u01aa\u0001\u0000\u0000\u0000\u000e\u01ac\u0001\u0000\u0000"+ - "\u0000\u000e\u01ae\u0001\u0000\u0000\u0000\u000e\u01b0\u0001\u0000\u0000"+ + "\u0000\u000e\u01ae\u0001\u0000\u0000\u0000\u000f\u01b0\u0001\u0000\u0000"+ "\u0000\u000f\u01b2\u0001\u0000\u0000\u0000\u000f\u01b4\u0001\u0000\u0000"+ "\u0000\u000f\u01b6\u0001\u0000\u0000\u0000\u000f\u01b8\u0001\u0000\u0000"+ "\u0000\u000f\u01ba\u0001\u0000\u0000\u0000\u000f\u01bc\u0001\u0000\u0000"+ "\u0000\u000f\u01be\u0001\u0000\u0000\u0000\u000f\u01c0\u0001\u0000\u0000"+ - "\u0000\u000f\u01c2\u0001\u0000\u0000\u0000\u0010\u01c4\u0001\u0000\u0000"+ - "\u0000\u0012\u01ce\u0001\u0000\u0000\u0000\u0014\u01d5\u0001\u0000\u0000"+ - "\u0000\u0016\u01de\u0001\u0000\u0000\u0000\u0018\u01e5\u0001\u0000\u0000"+ - "\u0000\u001a\u01ef\u0001\u0000\u0000\u0000\u001c\u01f6\u0001\u0000\u0000"+ - "\u0000\u001e\u01fd\u0001\u0000\u0000\u0000 \u0204\u0001\u0000\u0000\u0000"+ - "\"\u020c\u0001\u0000\u0000\u0000$\u0218\u0001\u0000\u0000\u0000&\u0221"+ - "\u0001\u0000\u0000\u0000(\u0227\u0001\u0000\u0000\u0000*\u022e\u0001\u0000"+ - "\u0000\u0000,\u0235\u0001\u0000\u0000\u0000.\u023d\u0001\u0000\u0000\u0000"+ - "0\u0245\u0001\u0000\u0000\u00002\u0254\u0001\u0000\u0000\u00004\u0260"+ - "\u0001\u0000\u0000\u00006\u026b\u0001\u0000\u0000\u00008\u0273\u0001\u0000"+ - "\u0000\u0000:\u027b\u0001\u0000\u0000\u0000<\u0283\u0001\u0000\u0000\u0000"+ - ">\u028c\u0001\u0000\u0000\u0000@\u0297\u0001\u0000\u0000\u0000B\u029d"+ - "\u0001\u0000\u0000\u0000D\u02ae\u0001\u0000\u0000\u0000F\u02be\u0001\u0000"+ - "\u0000\u0000H\u02c4\u0001\u0000\u0000\u0000J\u02c8\u0001\u0000\u0000\u0000"+ - "L\u02ca\u0001\u0000\u0000\u0000N\u02cc\u0001\u0000\u0000\u0000P\u02cf"+ - "\u0001\u0000\u0000\u0000R\u02d1\u0001\u0000\u0000\u0000T\u02da\u0001\u0000"+ - "\u0000\u0000V\u02dc\u0001\u0000\u0000\u0000X\u02e1\u0001\u0000\u0000\u0000"+ - "Z\u02e3\u0001\u0000\u0000\u0000\\\u02e8\u0001\u0000\u0000\u0000^\u0307"+ - "\u0001\u0000\u0000\u0000`\u030a\u0001\u0000\u0000\u0000b\u0338\u0001\u0000"+ - "\u0000\u0000d\u033a\u0001\u0000\u0000\u0000f\u033d\u0001\u0000\u0000\u0000"+ - "h\u0341\u0001\u0000\u0000\u0000j\u0345\u0001\u0000\u0000\u0000l\u0347"+ - "\u0001\u0000\u0000\u0000n\u034a\u0001\u0000\u0000\u0000p\u034c\u0001\u0000"+ - "\u0000\u0000r\u034e\u0001\u0000\u0000\u0000t\u0353\u0001\u0000\u0000\u0000"+ - "v\u0355\u0001\u0000\u0000\u0000x\u035b\u0001\u0000\u0000\u0000z\u0361"+ - "\u0001\u0000\u0000\u0000|\u0364\u0001\u0000\u0000\u0000~\u0367\u0001\u0000"+ - "\u0000\u0000\u0080\u036c\u0001\u0000\u0000\u0000\u0082\u0371\u0001\u0000"+ - "\u0000\u0000\u0084\u0373\u0001\u0000\u0000\u0000\u0086\u0377\u0001\u0000"+ - "\u0000\u0000\u0088\u037c\u0001\u0000\u0000\u0000\u008a\u0382\u0001\u0000"+ - "\u0000\u0000\u008c\u0385\u0001\u0000\u0000\u0000\u008e\u0387\u0001\u0000"+ - "\u0000\u0000\u0090\u038d\u0001\u0000\u0000\u0000\u0092\u038f\u0001\u0000"+ - "\u0000\u0000\u0094\u0394\u0001\u0000\u0000\u0000\u0096\u0397\u0001\u0000"+ - "\u0000\u0000\u0098\u039a\u0001\u0000\u0000\u0000\u009a\u039d\u0001\u0000"+ - "\u0000\u0000\u009c\u039f\u0001\u0000\u0000\u0000\u009e\u03a2\u0001\u0000"+ - "\u0000\u0000\u00a0\u03a4\u0001\u0000\u0000\u0000\u00a2\u03a7\u0001\u0000"+ - "\u0000\u0000\u00a4\u03a9\u0001\u0000\u0000\u0000\u00a6\u03ab\u0001\u0000"+ - "\u0000\u0000\u00a8\u03ad\u0001\u0000\u0000\u0000\u00aa\u03af\u0001\u0000"+ - "\u0000\u0000\u00ac\u03b1\u0001\u0000\u0000\u0000\u00ae\u03b3\u0001\u0000"+ - "\u0000\u0000\u00b0\u03b5\u0001\u0000\u0000\u0000\u00b2\u03ca\u0001\u0000"+ - "\u0000\u0000\u00b4\u03cc\u0001\u0000\u0000\u0000\u00b6\u03d1\u0001\u0000"+ - "\u0000\u0000\u00b8\u03e6\u0001\u0000\u0000\u0000\u00ba\u03e8\u0001\u0000"+ - "\u0000\u0000\u00bc\u03f0\u0001\u0000\u0000\u0000\u00be\u03f2\u0001\u0000"+ - "\u0000\u0000\u00c0\u03f6\u0001\u0000\u0000\u0000\u00c2\u03fa\u0001\u0000"+ - "\u0000\u0000\u00c4\u03fe\u0001\u0000\u0000\u0000\u00c6\u0403\u0001\u0000"+ - "\u0000\u0000\u00c8\u0408\u0001\u0000\u0000\u0000\u00ca\u040c\u0001\u0000"+ - "\u0000\u0000\u00cc\u0410\u0001\u0000\u0000\u0000\u00ce\u0414\u0001\u0000"+ - "\u0000\u0000\u00d0\u0419\u0001\u0000\u0000\u0000\u00d2\u041d\u0001\u0000"+ - "\u0000\u0000\u00d4\u0421\u0001\u0000\u0000\u0000\u00d6\u0425\u0001\u0000"+ - "\u0000\u0000\u00d8\u0429\u0001\u0000\u0000\u0000\u00da\u042d\u0001\u0000"+ - "\u0000\u0000\u00dc\u0439\u0001\u0000\u0000\u0000\u00de\u043c\u0001\u0000"+ - "\u0000\u0000\u00e0\u0440\u0001\u0000\u0000\u0000\u00e2\u0444\u0001\u0000"+ - "\u0000\u0000\u00e4\u0448\u0001\u0000\u0000\u0000\u00e6\u044c\u0001\u0000"+ - "\u0000\u0000\u00e8\u0450\u0001\u0000\u0000\u0000\u00ea\u0454\u0001\u0000"+ - "\u0000\u0000\u00ec\u0459\u0001\u0000\u0000\u0000\u00ee\u045d\u0001\u0000"+ - "\u0000\u0000\u00f0\u0461\u0001\u0000\u0000\u0000\u00f2\u0466\u0001\u0000"+ - "\u0000\u0000\u00f4\u046f\u0001\u0000\u0000\u0000\u00f6\u0484\u0001\u0000"+ - "\u0000\u0000\u00f8\u0488\u0001\u0000\u0000\u0000\u00fa\u048c\u0001\u0000"+ - "\u0000\u0000\u00fc\u0490\u0001\u0000\u0000\u0000\u00fe\u0494\u0001\u0000"+ - "\u0000\u0000\u0100\u0498\u0001\u0000\u0000\u0000\u0102\u049d\u0001\u0000"+ - "\u0000\u0000\u0104\u04a1\u0001\u0000\u0000\u0000\u0106\u04a5\u0001\u0000"+ - "\u0000\u0000\u0108\u04a9\u0001\u0000\u0000\u0000\u010a\u04ae\u0001\u0000"+ - "\u0000\u0000\u010c\u04b3\u0001\u0000\u0000\u0000\u010e\u04b6\u0001\u0000"+ - "\u0000\u0000\u0110\u04ba\u0001\u0000\u0000\u0000\u0112\u04be\u0001\u0000"+ - "\u0000\u0000\u0114\u04c2\u0001\u0000\u0000\u0000\u0116\u04c6\u0001\u0000"+ - "\u0000\u0000\u0118\u04cb\u0001\u0000\u0000\u0000\u011a\u04d0\u0001\u0000"+ - "\u0000\u0000\u011c\u04d5\u0001\u0000\u0000\u0000\u011e\u04dc\u0001\u0000"+ - "\u0000\u0000\u0120\u04e5\u0001\u0000\u0000\u0000\u0122\u04ec\u0001\u0000"+ - "\u0000\u0000\u0124\u04f0\u0001\u0000\u0000\u0000\u0126\u04f4\u0001\u0000"+ - "\u0000\u0000\u0128\u04f8\u0001\u0000\u0000\u0000\u012a\u04fc\u0001\u0000"+ - "\u0000\u0000\u012c\u0502\u0001\u0000\u0000\u0000\u012e\u0506\u0001\u0000"+ - "\u0000\u0000\u0130\u050a\u0001\u0000\u0000\u0000\u0132\u050e\u0001\u0000"+ - "\u0000\u0000\u0134\u0512\u0001\u0000\u0000\u0000\u0136\u0516\u0001\u0000"+ - "\u0000\u0000\u0138\u051a\u0001\u0000\u0000\u0000\u013a\u051f\u0001\u0000"+ - "\u0000\u0000\u013c\u0524\u0001\u0000\u0000\u0000\u013e\u0528\u0001\u0000"+ - "\u0000\u0000\u0140\u052c\u0001\u0000\u0000\u0000\u0142\u0530\u0001\u0000"+ - "\u0000\u0000\u0144\u0535\u0001\u0000\u0000\u0000\u0146\u0539\u0001\u0000"+ - "\u0000\u0000\u0148\u053e\u0001\u0000\u0000\u0000\u014a\u0543\u0001\u0000"+ - "\u0000\u0000\u014c\u0547\u0001\u0000\u0000\u0000\u014e\u054b\u0001\u0000"+ - "\u0000\u0000\u0150\u054f\u0001\u0000\u0000\u0000\u0152\u0553\u0001\u0000"+ - "\u0000\u0000\u0154\u0557\u0001\u0000\u0000\u0000\u0156\u055c\u0001\u0000"+ - "\u0000\u0000\u0158\u0561\u0001\u0000\u0000\u0000\u015a\u0565\u0001\u0000"+ - "\u0000\u0000\u015c\u0569\u0001\u0000\u0000\u0000\u015e\u056d\u0001\u0000"+ - "\u0000\u0000\u0160\u0572\u0001\u0000\u0000\u0000\u0162\u057b\u0001\u0000"+ - "\u0000\u0000\u0164\u057f\u0001\u0000\u0000\u0000\u0166\u0583\u0001\u0000"+ - "\u0000\u0000\u0168\u0587\u0001\u0000\u0000\u0000\u016a\u058b\u0001\u0000"+ - "\u0000\u0000\u016c\u0590\u0001\u0000\u0000\u0000\u016e\u0594\u0001\u0000"+ - "\u0000\u0000\u0170\u0598\u0001\u0000\u0000\u0000\u0172\u059c\u0001\u0000"+ - "\u0000\u0000\u0174\u05a1\u0001\u0000\u0000\u0000\u0176\u05a5\u0001\u0000"+ - "\u0000\u0000\u0178\u05a9\u0001\u0000\u0000\u0000\u017a\u05ad\u0001\u0000"+ - "\u0000\u0000\u017c\u05b1\u0001\u0000\u0000\u0000\u017e\u05b5\u0001\u0000"+ - "\u0000\u0000\u0180\u05bb\u0001\u0000\u0000\u0000\u0182\u05bf\u0001\u0000"+ - "\u0000\u0000\u0184\u05c3\u0001\u0000\u0000\u0000\u0186\u05c7\u0001\u0000"+ - "\u0000\u0000\u0188\u05cb\u0001\u0000\u0000\u0000\u018a\u05cf\u0001\u0000"+ - "\u0000\u0000\u018c\u05d3\u0001\u0000\u0000\u0000\u018e\u05d8\u0001\u0000"+ - "\u0000\u0000\u0190\u05dc\u0001\u0000\u0000\u0000\u0192\u05e0\u0001\u0000"+ - "\u0000\u0000\u0194\u05e6\u0001\u0000\u0000\u0000\u0196\u05ef\u0001\u0000"+ - "\u0000\u0000\u0198\u05f3\u0001\u0000\u0000\u0000\u019a\u05f7\u0001\u0000"+ - "\u0000\u0000\u019c\u05fb\u0001\u0000\u0000\u0000\u019e\u05ff\u0001\u0000"+ - "\u0000\u0000\u01a0\u0603\u0001\u0000\u0000\u0000\u01a2\u0607\u0001\u0000"+ - "\u0000\u0000\u01a4\u060b\u0001\u0000\u0000\u0000\u01a6\u060f\u0001\u0000"+ - "\u0000\u0000\u01a8\u0614\u0001\u0000\u0000\u0000\u01aa\u061a\u0001\u0000"+ - "\u0000\u0000\u01ac\u0620\u0001\u0000\u0000\u0000\u01ae\u0624\u0001\u0000"+ - "\u0000\u0000\u01b0\u0628\u0001\u0000\u0000\u0000\u01b2\u062c\u0001\u0000"+ - "\u0000\u0000\u01b4\u0632\u0001\u0000\u0000\u0000\u01b6\u0638\u0001\u0000"+ - "\u0000\u0000\u01b8\u063c\u0001\u0000\u0000\u0000\u01ba\u0640\u0001\u0000"+ - "\u0000\u0000\u01bc\u0644\u0001\u0000\u0000\u0000\u01be\u064a\u0001\u0000"+ - "\u0000\u0000\u01c0\u0650\u0001\u0000\u0000\u0000\u01c2\u0656\u0001\u0000"+ - "\u0000\u0000\u01c4\u01c5\u0007\u0000\u0000\u0000\u01c5\u01c6\u0007\u0001"+ - "\u0000\u0000\u01c6\u01c7\u0007\u0002\u0000\u0000\u01c7\u01c8\u0007\u0002"+ - "\u0000\u0000\u01c8\u01c9\u0007\u0003\u0000\u0000\u01c9\u01ca\u0007\u0004"+ - "\u0000\u0000\u01ca\u01cb\u0007\u0005\u0000\u0000\u01cb\u01cc\u0001\u0000"+ - "\u0000\u0000\u01cc\u01cd\u0006\u0000\u0000\u0000\u01cd\u0011\u0001\u0000"+ - "\u0000\u0000\u01ce\u01cf\u0007\u0000\u0000\u0000\u01cf\u01d0\u0007\u0006"+ - "\u0000\u0000\u01d0\u01d1\u0007\u0007\u0000\u0000\u01d1\u01d2\u0007\b\u0000"+ - "\u0000\u01d2\u01d3\u0001\u0000\u0000\u0000\u01d3\u01d4\u0006\u0001\u0001"+ - "\u0000\u01d4\u0013\u0001\u0000\u0000\u0000\u01d5\u01d6\u0007\u0003\u0000"+ - "\u0000\u01d6\u01d7\u0007\t\u0000\u0000\u01d7\u01d8\u0007\u0006\u0000\u0000"+ - "\u01d8\u01d9\u0007\u0001\u0000\u0000\u01d9\u01da\u0007\u0004\u0000\u0000"+ - "\u01da\u01db\u0007\n\u0000\u0000\u01db\u01dc\u0001\u0000\u0000\u0000\u01dc"+ - "\u01dd\u0006\u0002\u0002\u0000\u01dd\u0015\u0001\u0000\u0000\u0000\u01de"+ - "\u01df\u0007\u0003\u0000\u0000\u01df\u01e0\u0007\u000b\u0000\u0000\u01e0"+ - "\u01e1\u0007\f\u0000\u0000\u01e1\u01e2\u0007\r\u0000\u0000\u01e2\u01e3"+ - "\u0001\u0000\u0000\u0000\u01e3\u01e4\u0006\u0003\u0000\u0000\u01e4\u0017"+ - "\u0001\u0000\u0000\u0000\u01e5\u01e6\u0007\u0003\u0000\u0000\u01e6\u01e7"+ - "\u0007\u000e\u0000\u0000\u01e7\u01e8\u0007\b\u0000\u0000\u01e8\u01e9\u0007"+ - "\r\u0000\u0000\u01e9\u01ea\u0007\f\u0000\u0000\u01ea\u01eb\u0007\u0001"+ - "\u0000\u0000\u01eb\u01ec\u0007\t\u0000\u0000\u01ec\u01ed\u0001\u0000\u0000"+ - "\u0000\u01ed\u01ee\u0006\u0004\u0003\u0000\u01ee\u0019\u0001\u0000\u0000"+ - "\u0000\u01ef\u01f0\u0007\u000f\u0000\u0000\u01f0\u01f1\u0007\u0006\u0000"+ - "\u0000\u01f1\u01f2\u0007\u0007\u0000\u0000\u01f2\u01f3\u0007\u0010\u0000"+ - "\u0000\u01f3\u01f4\u0001\u0000\u0000\u0000\u01f4\u01f5\u0006\u0005\u0004"+ - "\u0000\u01f5\u001b\u0001\u0000\u0000\u0000\u01f6\u01f7\u0007\u0011\u0000"+ - "\u0000\u01f7\u01f8\u0007\u0006\u0000\u0000\u01f8\u01f9\u0007\u0007\u0000"+ - "\u0000\u01f9\u01fa\u0007\u0012\u0000\u0000\u01fa\u01fb\u0001\u0000\u0000"+ - "\u0000\u01fb\u01fc\u0006\u0006\u0000\u0000\u01fc\u001d\u0001\u0000\u0000"+ - "\u0000\u01fd\u01fe\u0007\u0012\u0000\u0000\u01fe\u01ff\u0007\u0003\u0000"+ - "\u0000\u01ff\u0200\u0007\u0003\u0000\u0000\u0200\u0201\u0007\b\u0000\u0000"+ - "\u0201\u0202\u0001\u0000\u0000\u0000\u0202\u0203\u0006\u0007\u0001\u0000"+ - "\u0203\u001f\u0001\u0000\u0000\u0000\u0204\u0205\u0007\r\u0000\u0000\u0205"+ - "\u0206\u0007\u0001\u0000\u0000\u0206\u0207\u0007\u0010\u0000\u0000\u0207"+ - "\u0208\u0007\u0001\u0000\u0000\u0208\u0209\u0007\u0005\u0000\u0000\u0209"+ - "\u020a\u0001\u0000\u0000\u0000\u020a\u020b\u0006\b\u0000\u0000\u020b!"+ - "\u0001\u0000\u0000\u0000\u020c\u020d\u0007\u0010\u0000\u0000\u020d\u020e"+ - "\u0007\u000b\u0000\u0000\u020e\u020f\u0005_\u0000\u0000\u020f\u0210\u0007"+ - "\u0003\u0000\u0000\u0210\u0211\u0007\u000e\u0000\u0000\u0211\u0212\u0007"+ - "\b\u0000\u0000\u0212\u0213\u0007\f\u0000\u0000\u0213\u0214\u0007\t\u0000"+ - "\u0000\u0214\u0215\u0007\u0000\u0000\u0000\u0215\u0216\u0001\u0000\u0000"+ - "\u0000\u0216\u0217\u0006\t\u0005\u0000\u0217#\u0001\u0000\u0000\u0000"+ - "\u0218\u0219\u0007\u0006\u0000\u0000\u0219\u021a\u0007\u0003\u0000\u0000"+ - "\u021a\u021b\u0007\t\u0000\u0000\u021b\u021c\u0007\f\u0000\u0000\u021c"+ - "\u021d\u0007\u0010\u0000\u0000\u021d\u021e\u0007\u0003\u0000\u0000\u021e"+ - "\u021f\u0001\u0000\u0000\u0000\u021f\u0220\u0006\n\u0006\u0000\u0220%"+ - "\u0001\u0000\u0000\u0000\u0221\u0222\u0007\u0006\u0000\u0000\u0222\u0223"+ - "\u0007\u0007\u0000\u0000\u0223\u0224\u0007\u0013\u0000\u0000\u0224\u0225"+ - "\u0001\u0000\u0000\u0000\u0225\u0226\u0006\u000b\u0000\u0000\u0226\'\u0001"+ - "\u0000\u0000\u0000\u0227\u0228\u0007\u0002\u0000\u0000\u0228\u0229\u0007"+ - "\n\u0000\u0000\u0229\u022a\u0007\u0007\u0000\u0000\u022a\u022b\u0007\u0013"+ - "\u0000\u0000\u022b\u022c\u0001\u0000\u0000\u0000\u022c\u022d\u0006\f\u0007"+ - "\u0000\u022d)\u0001\u0000\u0000\u0000\u022e\u022f\u0007\u0002\u0000\u0000"+ - "\u022f\u0230\u0007\u0007\u0000\u0000\u0230\u0231\u0007\u0006\u0000\u0000"+ - "\u0231\u0232\u0007\u0005\u0000\u0000\u0232\u0233\u0001\u0000\u0000\u0000"+ - "\u0233\u0234\u0006\r\u0000\u0000\u0234+\u0001\u0000\u0000\u0000\u0235"+ - "\u0236\u0007\u0002\u0000\u0000\u0236\u0237\u0007\u0005\u0000\u0000\u0237"+ - "\u0238\u0007\f\u0000\u0000\u0238\u0239\u0007\u0005\u0000\u0000\u0239\u023a"+ - "\u0007\u0002\u0000\u0000\u023a\u023b\u0001\u0000\u0000\u0000\u023b\u023c"+ - "\u0006\u000e\u0000\u0000\u023c-\u0001\u0000\u0000\u0000\u023d\u023e\u0007"+ - "\u0013\u0000\u0000\u023e\u023f\u0007\n\u0000\u0000\u023f\u0240\u0007\u0003"+ - "\u0000\u0000\u0240\u0241\u0007\u0006\u0000\u0000\u0241\u0242\u0007\u0003"+ - "\u0000\u0000\u0242\u0243\u0001\u0000\u0000\u0000\u0243\u0244\u0006\u000f"+ - "\u0000\u0000\u0244/\u0001\u0000\u0000\u0000\u0245\u0246\u0004\u0010\u0000"+ - "\u0000\u0246\u0247\u0007\u0001\u0000\u0000\u0247\u0248\u0007\t\u0000\u0000"+ - "\u0248\u0249\u0007\r\u0000\u0000\u0249\u024a\u0007\u0001\u0000\u0000\u024a"+ - "\u024b\u0007\t\u0000\u0000\u024b\u024c\u0007\u0003\u0000\u0000\u024c\u024d"+ - "\u0007\u0002\u0000\u0000\u024d\u024e\u0007\u0005\u0000\u0000\u024e\u024f"+ - "\u0007\f\u0000\u0000\u024f\u0250\u0007\u0005\u0000\u0000\u0250\u0251\u0007"+ - "\u0002\u0000\u0000\u0251\u0252\u0001\u0000\u0000\u0000\u0252\u0253\u0006"+ - "\u0010\u0000\u0000\u02531\u0001\u0000\u0000\u0000\u0254\u0255\u0004\u0011"+ - "\u0001\u0000\u0255\u0256\u0007\r\u0000\u0000\u0256\u0257\u0007\u0007\u0000"+ - "\u0000\u0257\u0258\u0007\u0007\u0000\u0000\u0258\u0259\u0007\u0012\u0000"+ - "\u0000\u0259\u025a\u0007\u0014\u0000\u0000\u025a\u025b\u0007\b\u0000\u0000"+ - "\u025b\u025c\u0005_\u0000\u0000\u025c\u025d\u0005\u8001\uf414\u0000\u0000"+ - "\u025d\u025e\u0001\u0000\u0000\u0000\u025e\u025f\u0006\u0011\b\u0000\u025f"+ - "3\u0001\u0000\u0000\u0000\u0260\u0261\u0004\u0012\u0002\u0000\u0261\u0262"+ - "\u0007\u0010\u0000\u0000\u0262\u0263\u0007\u0003\u0000\u0000\u0263\u0264"+ - "\u0007\u0005\u0000\u0000\u0264\u0265\u0007\u0006\u0000\u0000\u0265\u0266"+ - "\u0007\u0001\u0000\u0000\u0266\u0267\u0007\u0004\u0000\u0000\u0267\u0268"+ - "\u0007\u0002\u0000\u0000\u0268\u0269\u0001\u0000\u0000\u0000\u0269\u026a"+ - "\u0006\u0012\t\u0000\u026a5\u0001\u0000\u0000\u0000\u026b\u026c\u0004"+ - "\u0013\u0003\u0000\u026c\u026d\u0007\u0015\u0000\u0000\u026d\u026e\u0007"+ - "\u0007\u0000\u0000\u026e\u026f\u0007\u0001\u0000\u0000\u026f\u0270\u0007"+ - "\t\u0000\u0000\u0270\u0271\u0001\u0000\u0000\u0000\u0271\u0272\u0006\u0013"+ - "\n\u0000\u02727\u0001\u0000\u0000\u0000\u0273\u0274\u0004\u0014\u0004"+ - "\u0000\u0274\u0275\u0007\u000f\u0000\u0000\u0275\u0276\u0007\u0014\u0000"+ - "\u0000\u0276\u0277\u0007\r\u0000\u0000\u0277\u0278\u0007\r\u0000\u0000"+ - "\u0278\u0279\u0001\u0000\u0000\u0000\u0279\u027a\u0006\u0014\n\u0000\u027a"+ - "9\u0001\u0000\u0000\u0000\u027b\u027c\u0004\u0015\u0005\u0000\u027c\u027d"+ - "\u0007\r\u0000\u0000\u027d\u027e\u0007\u0003\u0000\u0000\u027e\u027f\u0007"+ - "\u000f\u0000\u0000\u027f\u0280\u0007\u0005\u0000\u0000\u0280\u0281\u0001"+ - "\u0000\u0000\u0000\u0281\u0282\u0006\u0015\n\u0000\u0282;\u0001\u0000"+ - "\u0000\u0000\u0283\u0284\u0004\u0016\u0006\u0000\u0284\u0285\u0007\u0006"+ - "\u0000\u0000\u0285\u0286\u0007\u0001\u0000\u0000\u0286\u0287\u0007\u0011"+ - "\u0000\u0000\u0287\u0288\u0007\n\u0000\u0000\u0288\u0289\u0007\u0005\u0000"+ - "\u0000\u0289\u028a\u0001\u0000\u0000\u0000\u028a\u028b\u0006\u0016\n\u0000"+ - "\u028b=\u0001\u0000\u0000\u0000\u028c\u028d\u0004\u0017\u0007\u0000\u028d"+ - "\u028e\u0007\r\u0000\u0000\u028e\u028f\u0007\u0007\u0000\u0000\u028f\u0290"+ - "\u0007\u0007\u0000\u0000\u0290\u0291\u0007\u0012\u0000\u0000\u0291\u0292"+ - "\u0007\u0014\u0000\u0000\u0292\u0293\u0007\b\u0000\u0000\u0293\u0294\u0001"+ - "\u0000\u0000\u0000\u0294\u0295\u0006\u0017\n\u0000\u0295?\u0001\u0000"+ - "\u0000\u0000\u0296\u0298\b\u0016\u0000\u0000\u0297\u0296\u0001\u0000\u0000"+ - "\u0000\u0298\u0299\u0001\u0000\u0000\u0000\u0299\u0297\u0001\u0000\u0000"+ - "\u0000\u0299\u029a\u0001\u0000\u0000\u0000\u029a\u029b\u0001\u0000\u0000"+ - "\u0000\u029b\u029c\u0006\u0018\u0000\u0000\u029cA\u0001\u0000\u0000\u0000"+ - "\u029d\u029e\u0005/\u0000\u0000\u029e\u029f\u0005/\u0000\u0000\u029f\u02a3"+ - "\u0001\u0000\u0000\u0000\u02a0\u02a2\b\u0017\u0000\u0000\u02a1\u02a0\u0001"+ - "\u0000\u0000\u0000\u02a2\u02a5\u0001\u0000\u0000\u0000\u02a3\u02a1\u0001"+ - "\u0000\u0000\u0000\u02a3\u02a4\u0001\u0000\u0000\u0000\u02a4\u02a7\u0001"+ - "\u0000\u0000\u0000\u02a5\u02a3\u0001\u0000\u0000\u0000\u02a6\u02a8\u0005"+ - "\r\u0000\u0000\u02a7\u02a6\u0001\u0000\u0000\u0000\u02a7\u02a8\u0001\u0000"+ - "\u0000\u0000\u02a8\u02aa\u0001\u0000\u0000\u0000\u02a9\u02ab\u0005\n\u0000"+ - "\u0000\u02aa\u02a9\u0001\u0000\u0000\u0000\u02aa\u02ab\u0001\u0000\u0000"+ - "\u0000\u02ab\u02ac\u0001\u0000\u0000\u0000\u02ac\u02ad\u0006\u0019\u000b"+ - "\u0000\u02adC\u0001\u0000\u0000\u0000\u02ae\u02af\u0005/\u0000\u0000\u02af"+ - "\u02b0\u0005*\u0000\u0000\u02b0\u02b5\u0001\u0000\u0000\u0000\u02b1\u02b4"+ - "\u0003D\u001a\u0000\u02b2\u02b4\t\u0000\u0000\u0000\u02b3\u02b1\u0001"+ - "\u0000\u0000\u0000\u02b3\u02b2\u0001\u0000\u0000\u0000\u02b4\u02b7\u0001"+ - "\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000\u0000\u0000\u02b5\u02b3\u0001"+ - "\u0000\u0000\u0000\u02b6\u02b8\u0001\u0000\u0000\u0000\u02b7\u02b5\u0001"+ - "\u0000\u0000\u0000\u02b8\u02b9\u0005*\u0000\u0000\u02b9\u02ba\u0005/\u0000"+ - "\u0000\u02ba\u02bb\u0001\u0000\u0000\u0000\u02bb\u02bc\u0006\u001a\u000b"+ - "\u0000\u02bcE\u0001\u0000\u0000\u0000\u02bd\u02bf\u0007\u0018\u0000\u0000"+ - "\u02be\u02bd\u0001\u0000\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000\u0000"+ - "\u02c0\u02be\u0001\u0000\u0000\u0000\u02c0\u02c1\u0001\u0000\u0000\u0000"+ - "\u02c1\u02c2\u0001\u0000\u0000\u0000\u02c2\u02c3\u0006\u001b\u000b\u0000"+ - "\u02c3G\u0001\u0000\u0000\u0000\u02c4\u02c5\u0005|\u0000\u0000\u02c5\u02c6"+ - "\u0001\u0000\u0000\u0000\u02c6\u02c7\u0006\u001c\f\u0000\u02c7I\u0001"+ - "\u0000\u0000\u0000\u02c8\u02c9\u0007\u0019\u0000\u0000\u02c9K\u0001\u0000"+ - "\u0000\u0000\u02ca\u02cb\u0007\u001a\u0000\u0000\u02cbM\u0001\u0000\u0000"+ - "\u0000\u02cc\u02cd\u0005\\\u0000\u0000\u02cd\u02ce\u0007\u001b\u0000\u0000"+ - "\u02ceO\u0001\u0000\u0000\u0000\u02cf\u02d0\b\u001c\u0000\u0000\u02d0"+ - "Q\u0001\u0000\u0000\u0000\u02d1\u02d3\u0007\u0003\u0000\u0000\u02d2\u02d4"+ - "\u0007\u001d\u0000\u0000\u02d3\u02d2\u0001\u0000\u0000\u0000\u02d3\u02d4"+ - "\u0001\u0000\u0000\u0000\u02d4\u02d6\u0001\u0000\u0000\u0000\u02d5\u02d7"+ - "\u0003J\u001d\u0000\u02d6\u02d5\u0001\u0000\u0000\u0000\u02d7\u02d8\u0001"+ - "\u0000\u0000\u0000\u02d8\u02d6\u0001\u0000\u0000\u0000\u02d8\u02d9\u0001"+ - "\u0000\u0000\u0000\u02d9S\u0001\u0000\u0000\u0000\u02da\u02db\u0005@\u0000"+ - "\u0000\u02dbU\u0001\u0000\u0000\u0000\u02dc\u02dd\u0005`\u0000\u0000\u02dd"+ - "W\u0001\u0000\u0000\u0000\u02de\u02e2\b\u001e\u0000\u0000\u02df\u02e0"+ - "\u0005`\u0000\u0000\u02e0\u02e2\u0005`\u0000\u0000\u02e1\u02de\u0001\u0000"+ - "\u0000\u0000\u02e1\u02df\u0001\u0000\u0000\u0000\u02e2Y\u0001\u0000\u0000"+ - "\u0000\u02e3\u02e4\u0005_\u0000\u0000\u02e4[\u0001\u0000\u0000\u0000\u02e5"+ - "\u02e9\u0003L\u001e\u0000\u02e6\u02e9\u0003J\u001d\u0000\u02e7\u02e9\u0003"+ - "Z%\u0000\u02e8\u02e5\u0001\u0000\u0000\u0000\u02e8\u02e6\u0001\u0000\u0000"+ - "\u0000\u02e8\u02e7\u0001\u0000\u0000\u0000\u02e9]\u0001\u0000\u0000\u0000"+ - "\u02ea\u02ef\u0005\"\u0000\u0000\u02eb\u02ee\u0003N\u001f\u0000\u02ec"+ - "\u02ee\u0003P \u0000\u02ed\u02eb\u0001\u0000\u0000\u0000\u02ed\u02ec\u0001"+ - "\u0000\u0000\u0000\u02ee\u02f1\u0001\u0000\u0000\u0000\u02ef\u02ed\u0001"+ - "\u0000\u0000\u0000\u02ef\u02f0\u0001\u0000\u0000\u0000\u02f0\u02f2\u0001"+ - "\u0000\u0000\u0000\u02f1\u02ef\u0001\u0000\u0000\u0000\u02f2\u0308\u0005"+ + "\u0000\u0010\u01c2\u0001\u0000\u0000\u0000\u0012\u01cc\u0001\u0000\u0000"+ + "\u0000\u0014\u01d3\u0001\u0000\u0000\u0000\u0016\u01dc\u0001\u0000\u0000"+ + "\u0000\u0018\u01e3\u0001\u0000\u0000\u0000\u001a\u01ed\u0001\u0000\u0000"+ + "\u0000\u001c\u01f4\u0001\u0000\u0000\u0000\u001e\u01fb\u0001\u0000\u0000"+ + "\u0000 \u0202\u0001\u0000\u0000\u0000\"\u020a\u0001\u0000\u0000\u0000"+ + "$\u0216\u0001\u0000\u0000\u0000&\u021f\u0001\u0000\u0000\u0000(\u0225"+ + "\u0001\u0000\u0000\u0000*\u022c\u0001\u0000\u0000\u0000,\u0233\u0001\u0000"+ + "\u0000\u0000.\u023b\u0001\u0000\u0000\u00000\u0243\u0001\u0000\u0000\u0000"+ + "2\u024c\u0001\u0000\u0000\u00004\u025b\u0001\u0000\u0000\u00006\u0267"+ + "\u0001\u0000\u0000\u00008\u0272\u0001\u0000\u0000\u0000:\u027a\u0001\u0000"+ + "\u0000\u0000<\u0282\u0001\u0000\u0000\u0000>\u028c\u0001\u0000\u0000\u0000"+ + "@\u0292\u0001\u0000\u0000\u0000B\u02a3\u0001\u0000\u0000\u0000D\u02b3"+ + "\u0001\u0000\u0000\u0000F\u02b9\u0001\u0000\u0000\u0000H\u02bd\u0001\u0000"+ + "\u0000\u0000J\u02bf\u0001\u0000\u0000\u0000L\u02c1\u0001\u0000\u0000\u0000"+ + "N\u02c4\u0001\u0000\u0000\u0000P\u02c6\u0001\u0000\u0000\u0000R\u02cf"+ + "\u0001\u0000\u0000\u0000T\u02d1\u0001\u0000\u0000\u0000V\u02d6\u0001\u0000"+ + "\u0000\u0000X\u02d8\u0001\u0000\u0000\u0000Z\u02dd\u0001\u0000\u0000\u0000"+ + "\\\u02fc\u0001\u0000\u0000\u0000^\u02ff\u0001\u0000\u0000\u0000`\u032d"+ + "\u0001\u0000\u0000\u0000b\u032f\u0001\u0000\u0000\u0000d\u0332\u0001\u0000"+ + "\u0000\u0000f\u0336\u0001\u0000\u0000\u0000h\u033a\u0001\u0000\u0000\u0000"+ + "j\u033c\u0001\u0000\u0000\u0000l\u033f\u0001\u0000\u0000\u0000n\u0341"+ + "\u0001\u0000\u0000\u0000p\u0343\u0001\u0000\u0000\u0000r\u0348\u0001\u0000"+ + "\u0000\u0000t\u034a\u0001\u0000\u0000\u0000v\u0350\u0001\u0000\u0000\u0000"+ + "x\u0356\u0001\u0000\u0000\u0000z\u0359\u0001\u0000\u0000\u0000|\u035c"+ + "\u0001\u0000\u0000\u0000~\u0361\u0001\u0000\u0000\u0000\u0080\u0366\u0001"+ + "\u0000\u0000\u0000\u0082\u0368\u0001\u0000\u0000\u0000\u0084\u036c\u0001"+ + "\u0000\u0000\u0000\u0086\u0371\u0001\u0000\u0000\u0000\u0088\u0377\u0001"+ + "\u0000\u0000\u0000\u008a\u037a\u0001\u0000\u0000\u0000\u008c\u037c\u0001"+ + "\u0000\u0000\u0000\u008e\u0382\u0001\u0000\u0000\u0000\u0090\u0384\u0001"+ + "\u0000\u0000\u0000\u0092\u0389\u0001\u0000\u0000\u0000\u0094\u038c\u0001"+ + "\u0000\u0000\u0000\u0096\u038f\u0001\u0000\u0000\u0000\u0098\u0392\u0001"+ + "\u0000\u0000\u0000\u009a\u0394\u0001\u0000\u0000\u0000\u009c\u0397\u0001"+ + "\u0000\u0000\u0000\u009e\u0399\u0001\u0000\u0000\u0000\u00a0\u039c\u0001"+ + "\u0000\u0000\u0000\u00a2\u039e\u0001\u0000\u0000\u0000\u00a4\u03a0\u0001"+ + "\u0000\u0000\u0000\u00a6\u03a2\u0001\u0000\u0000\u0000\u00a8\u03a4\u0001"+ + "\u0000\u0000\u0000\u00aa\u03a6\u0001\u0000\u0000\u0000\u00ac\u03a8\u0001"+ + "\u0000\u0000\u0000\u00ae\u03aa\u0001\u0000\u0000\u0000\u00b0\u03bf\u0001"+ + "\u0000\u0000\u0000\u00b2\u03c1\u0001\u0000\u0000\u0000\u00b4\u03c6\u0001"+ + "\u0000\u0000\u0000\u00b6\u03db\u0001\u0000\u0000\u0000\u00b8\u03dd\u0001"+ + "\u0000\u0000\u0000\u00ba\u03e5\u0001\u0000\u0000\u0000\u00bc\u03e7\u0001"+ + "\u0000\u0000\u0000\u00be\u03eb\u0001\u0000\u0000\u0000\u00c0\u03ef\u0001"+ + "\u0000\u0000\u0000\u00c2\u03f3\u0001\u0000\u0000\u0000\u00c4\u03f8\u0001"+ + "\u0000\u0000\u0000\u00c6\u03fd\u0001\u0000\u0000\u0000\u00c8\u0401\u0001"+ + "\u0000\u0000\u0000\u00ca\u0405\u0001\u0000\u0000\u0000\u00cc\u0409\u0001"+ + "\u0000\u0000\u0000\u00ce\u040e\u0001\u0000\u0000\u0000\u00d0\u0412\u0001"+ + "\u0000\u0000\u0000\u00d2\u0416\u0001\u0000\u0000\u0000\u00d4\u041a\u0001"+ + "\u0000\u0000\u0000\u00d6\u041e\u0001\u0000\u0000\u0000\u00d8\u0422\u0001"+ + "\u0000\u0000\u0000\u00da\u042e\u0001\u0000\u0000\u0000\u00dc\u0431\u0001"+ + "\u0000\u0000\u0000\u00de\u0435\u0001\u0000\u0000\u0000\u00e0\u0439\u0001"+ + "\u0000\u0000\u0000\u00e2\u043d\u0001\u0000\u0000\u0000\u00e4\u0441\u0001"+ + "\u0000\u0000\u0000\u00e6\u0445\u0001\u0000\u0000\u0000\u00e8\u0449\u0001"+ + "\u0000\u0000\u0000\u00ea\u044e\u0001\u0000\u0000\u0000\u00ec\u0452\u0001"+ + "\u0000\u0000\u0000\u00ee\u0456\u0001\u0000\u0000\u0000\u00f0\u045a\u0001"+ + "\u0000\u0000\u0000\u00f2\u0462\u0001\u0000\u0000\u0000\u00f4\u0477\u0001"+ + "\u0000\u0000\u0000\u00f6\u047b\u0001\u0000\u0000\u0000\u00f8\u047f\u0001"+ + "\u0000\u0000\u0000\u00fa\u0483\u0001\u0000\u0000\u0000\u00fc\u0487\u0001"+ + "\u0000\u0000\u0000\u00fe\u048b\u0001\u0000\u0000\u0000\u0100\u0490\u0001"+ + "\u0000\u0000\u0000\u0102\u0494\u0001\u0000\u0000\u0000\u0104\u0498\u0001"+ + "\u0000\u0000\u0000\u0106\u049c\u0001\u0000\u0000\u0000\u0108\u04a0\u0001"+ + "\u0000\u0000\u0000\u010a\u04a4\u0001\u0000\u0000\u0000\u010c\u04a7\u0001"+ + "\u0000\u0000\u0000\u010e\u04ab\u0001\u0000\u0000\u0000\u0110\u04af\u0001"+ + "\u0000\u0000\u0000\u0112\u04b3\u0001\u0000\u0000\u0000\u0114\u04b7\u0001"+ + "\u0000\u0000\u0000\u0116\u04bc\u0001\u0000\u0000\u0000\u0118\u04c1\u0001"+ + "\u0000\u0000\u0000\u011a\u04c6\u0001\u0000\u0000\u0000\u011c\u04cd\u0001"+ + "\u0000\u0000\u0000\u011e\u04d6\u0001\u0000\u0000\u0000\u0120\u04dd\u0001"+ + "\u0000\u0000\u0000\u0122\u04e1\u0001\u0000\u0000\u0000\u0124\u04e5\u0001"+ + "\u0000\u0000\u0000\u0126\u04e9\u0001\u0000\u0000\u0000\u0128\u04ed\u0001"+ + "\u0000\u0000\u0000\u012a\u04f3\u0001\u0000\u0000\u0000\u012c\u04f7\u0001"+ + "\u0000\u0000\u0000\u012e\u04fb\u0001\u0000\u0000\u0000\u0130\u04ff\u0001"+ + "\u0000\u0000\u0000\u0132\u0503\u0001\u0000\u0000\u0000\u0134\u0507\u0001"+ + "\u0000\u0000\u0000\u0136\u050b\u0001\u0000\u0000\u0000\u0138\u050f\u0001"+ + "\u0000\u0000\u0000\u013a\u0513\u0001\u0000\u0000\u0000\u013c\u0517\u0001"+ + "\u0000\u0000\u0000\u013e\u051b\u0001\u0000\u0000\u0000\u0140\u051f\u0001"+ + "\u0000\u0000\u0000\u0142\u0524\u0001\u0000\u0000\u0000\u0144\u0528\u0001"+ + "\u0000\u0000\u0000\u0146\u052c\u0001\u0000\u0000\u0000\u0148\u0530\u0001"+ + "\u0000\u0000\u0000\u014a\u0534\u0001\u0000\u0000\u0000\u014c\u0538\u0001"+ + "\u0000\u0000\u0000\u014e\u053c\u0001\u0000\u0000\u0000\u0150\u0540\u0001"+ + "\u0000\u0000\u0000\u0152\u0544\u0001\u0000\u0000\u0000\u0154\u0549\u0001"+ + "\u0000\u0000\u0000\u0156\u054e\u0001\u0000\u0000\u0000\u0158\u0552\u0001"+ + "\u0000\u0000\u0000\u015a\u0556\u0001\u0000\u0000\u0000\u015c\u055a\u0001"+ + "\u0000\u0000\u0000\u015e\u055f\u0001\u0000\u0000\u0000\u0160\u0568\u0001"+ + "\u0000\u0000\u0000\u0162\u056c\u0001\u0000\u0000\u0000\u0164\u0570\u0001"+ + "\u0000\u0000\u0000\u0166\u0574\u0001\u0000\u0000\u0000\u0168\u0578\u0001"+ + "\u0000\u0000\u0000\u016a\u057d\u0001\u0000\u0000\u0000\u016c\u0581\u0001"+ + "\u0000\u0000\u0000\u016e\u0585\u0001\u0000\u0000\u0000\u0170\u0589\u0001"+ + "\u0000\u0000\u0000\u0172\u058e\u0001\u0000\u0000\u0000\u0174\u0592\u0001"+ + "\u0000\u0000\u0000\u0176\u0596\u0001\u0000\u0000\u0000\u0178\u059a\u0001"+ + "\u0000\u0000\u0000\u017a\u059e\u0001\u0000\u0000\u0000\u017c\u05a2\u0001"+ + "\u0000\u0000\u0000\u017e\u05a8\u0001\u0000\u0000\u0000\u0180\u05ac\u0001"+ + "\u0000\u0000\u0000\u0182\u05b0\u0001\u0000\u0000\u0000\u0184\u05b4\u0001"+ + "\u0000\u0000\u0000\u0186\u05b8\u0001\u0000\u0000\u0000\u0188\u05bc\u0001"+ + "\u0000\u0000\u0000\u018a\u05c0\u0001\u0000\u0000\u0000\u018c\u05c5\u0001"+ + "\u0000\u0000\u0000\u018e\u05ca\u0001\u0000\u0000\u0000\u0190\u05ce\u0001"+ + "\u0000\u0000\u0000\u0192\u05d4\u0001\u0000\u0000\u0000\u0194\u05dd\u0001"+ + "\u0000\u0000\u0000\u0196\u05e1\u0001\u0000\u0000\u0000\u0198\u05e5\u0001"+ + "\u0000\u0000\u0000\u019a\u05e9\u0001\u0000\u0000\u0000\u019c\u05ed\u0001"+ + "\u0000\u0000\u0000\u019e\u05f1\u0001\u0000\u0000\u0000\u01a0\u05f5\u0001"+ + "\u0000\u0000\u0000\u01a2\u05f9\u0001\u0000\u0000\u0000\u01a4\u05fd\u0001"+ + "\u0000\u0000\u0000\u01a6\u0602\u0001\u0000\u0000\u0000\u01a8\u0608\u0001"+ + "\u0000\u0000\u0000\u01aa\u060e\u0001\u0000\u0000\u0000\u01ac\u0612\u0001"+ + "\u0000\u0000\u0000\u01ae\u0616\u0001\u0000\u0000\u0000\u01b0\u061a\u0001"+ + "\u0000\u0000\u0000\u01b2\u0620\u0001\u0000\u0000\u0000\u01b4\u0626\u0001"+ + "\u0000\u0000\u0000\u01b6\u062a\u0001\u0000\u0000\u0000\u01b8\u062e\u0001"+ + "\u0000\u0000\u0000\u01ba\u0632\u0001\u0000\u0000\u0000\u01bc\u0638\u0001"+ + "\u0000\u0000\u0000\u01be\u063e\u0001\u0000\u0000\u0000\u01c0\u0644\u0001"+ + "\u0000\u0000\u0000\u01c2\u01c3\u0007\u0000\u0000\u0000\u01c3\u01c4\u0007"+ + "\u0001\u0000\u0000\u01c4\u01c5\u0007\u0002\u0000\u0000\u01c5\u01c6\u0007"+ + "\u0002\u0000\u0000\u01c6\u01c7\u0007\u0003\u0000\u0000\u01c7\u01c8\u0007"+ + "\u0004\u0000\u0000\u01c8\u01c9\u0007\u0005\u0000\u0000\u01c9\u01ca\u0001"+ + "\u0000\u0000\u0000\u01ca\u01cb\u0006\u0000\u0000\u0000\u01cb\u0011\u0001"+ + "\u0000\u0000\u0000\u01cc\u01cd\u0007\u0000\u0000\u0000\u01cd\u01ce\u0007"+ + "\u0006\u0000\u0000\u01ce\u01cf\u0007\u0007\u0000\u0000\u01cf\u01d0\u0007"+ + "\b\u0000\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1\u01d2\u0006\u0001"+ + "\u0001\u0000\u01d2\u0013\u0001\u0000\u0000\u0000\u01d3\u01d4\u0007\u0003"+ + "\u0000\u0000\u01d4\u01d5\u0007\t\u0000\u0000\u01d5\u01d6\u0007\u0006\u0000"+ + "\u0000\u01d6\u01d7\u0007\u0001\u0000\u0000\u01d7\u01d8\u0007\u0004\u0000"+ + "\u0000\u01d8\u01d9\u0007\n\u0000\u0000\u01d9\u01da\u0001\u0000\u0000\u0000"+ + "\u01da\u01db\u0006\u0002\u0002\u0000\u01db\u0015\u0001\u0000\u0000\u0000"+ + "\u01dc\u01dd\u0007\u0003\u0000\u0000\u01dd\u01de\u0007\u000b\u0000\u0000"+ + "\u01de\u01df\u0007\f\u0000\u0000\u01df\u01e0\u0007\r\u0000\u0000\u01e0"+ + "\u01e1\u0001\u0000\u0000\u0000\u01e1\u01e2\u0006\u0003\u0000\u0000\u01e2"+ + "\u0017\u0001\u0000\u0000\u0000\u01e3\u01e4\u0007\u0003\u0000\u0000\u01e4"+ + "\u01e5\u0007\u000e\u0000\u0000\u01e5\u01e6\u0007\b\u0000\u0000\u01e6\u01e7"+ + "\u0007\r\u0000\u0000\u01e7\u01e8\u0007\f\u0000\u0000\u01e8\u01e9\u0007"+ + "\u0001\u0000\u0000\u01e9\u01ea\u0007\t\u0000\u0000\u01ea\u01eb\u0001\u0000"+ + "\u0000\u0000\u01eb\u01ec\u0006\u0004\u0003\u0000\u01ec\u0019\u0001\u0000"+ + "\u0000\u0000\u01ed\u01ee\u0007\u000f\u0000\u0000\u01ee\u01ef\u0007\u0006"+ + "\u0000\u0000\u01ef\u01f0\u0007\u0007\u0000\u0000\u01f0\u01f1\u0007\u0010"+ + "\u0000\u0000\u01f1\u01f2\u0001\u0000\u0000\u0000\u01f2\u01f3\u0006\u0005"+ + "\u0004\u0000\u01f3\u001b\u0001\u0000\u0000\u0000\u01f4\u01f5\u0007\u0011"+ + "\u0000\u0000\u01f5\u01f6\u0007\u0006\u0000\u0000\u01f6\u01f7\u0007\u0007"+ + "\u0000\u0000\u01f7\u01f8\u0007\u0012\u0000\u0000\u01f8\u01f9\u0001\u0000"+ + "\u0000\u0000\u01f9\u01fa\u0006\u0006\u0000\u0000\u01fa\u001d\u0001\u0000"+ + "\u0000\u0000\u01fb\u01fc\u0007\u0012\u0000\u0000\u01fc\u01fd\u0007\u0003"+ + "\u0000\u0000\u01fd\u01fe\u0007\u0003\u0000\u0000\u01fe\u01ff\u0007\b\u0000"+ + "\u0000\u01ff\u0200\u0001\u0000\u0000\u0000\u0200\u0201\u0006\u0007\u0001"+ + "\u0000\u0201\u001f\u0001\u0000\u0000\u0000\u0202\u0203\u0007\r\u0000\u0000"+ + "\u0203\u0204\u0007\u0001\u0000\u0000\u0204\u0205\u0007\u0010\u0000\u0000"+ + "\u0205\u0206\u0007\u0001\u0000\u0000\u0206\u0207\u0007\u0005\u0000\u0000"+ + "\u0207\u0208\u0001\u0000\u0000\u0000\u0208\u0209\u0006\b\u0000\u0000\u0209"+ + "!\u0001\u0000\u0000\u0000\u020a\u020b\u0007\u0010\u0000\u0000\u020b\u020c"+ + "\u0007\u000b\u0000\u0000\u020c\u020d\u0005_\u0000\u0000\u020d\u020e\u0007"+ + "\u0003\u0000\u0000\u020e\u020f\u0007\u000e\u0000\u0000\u020f\u0210\u0007"+ + "\b\u0000\u0000\u0210\u0211\u0007\f\u0000\u0000\u0211\u0212\u0007\t\u0000"+ + "\u0000\u0212\u0213\u0007\u0000\u0000\u0000\u0213\u0214\u0001\u0000\u0000"+ + "\u0000\u0214\u0215\u0006\t\u0005\u0000\u0215#\u0001\u0000\u0000\u0000"+ + "\u0216\u0217\u0007\u0006\u0000\u0000\u0217\u0218\u0007\u0003\u0000\u0000"+ + "\u0218\u0219\u0007\t\u0000\u0000\u0219\u021a\u0007\f\u0000\u0000\u021a"+ + "\u021b\u0007\u0010\u0000\u0000\u021b\u021c\u0007\u0003\u0000\u0000\u021c"+ + "\u021d\u0001\u0000\u0000\u0000\u021d\u021e\u0006\n\u0006\u0000\u021e%"+ + "\u0001\u0000\u0000\u0000\u021f\u0220\u0007\u0006\u0000\u0000\u0220\u0221"+ + "\u0007\u0007\u0000\u0000\u0221\u0222\u0007\u0013\u0000\u0000\u0222\u0223"+ + "\u0001\u0000\u0000\u0000\u0223\u0224\u0006\u000b\u0000\u0000\u0224\'\u0001"+ + "\u0000\u0000\u0000\u0225\u0226\u0007\u0002\u0000\u0000\u0226\u0227\u0007"+ + "\n\u0000\u0000\u0227\u0228\u0007\u0007\u0000\u0000\u0228\u0229\u0007\u0013"+ + "\u0000\u0000\u0229\u022a\u0001\u0000\u0000\u0000\u022a\u022b\u0006\f\u0007"+ + "\u0000\u022b)\u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0002\u0000\u0000"+ + "\u022d\u022e\u0007\u0007\u0000\u0000\u022e\u022f\u0007\u0006\u0000\u0000"+ + "\u022f\u0230\u0007\u0005\u0000\u0000\u0230\u0231\u0001\u0000\u0000\u0000"+ + "\u0231\u0232\u0006\r\u0000\u0000\u0232+\u0001\u0000\u0000\u0000\u0233"+ + "\u0234\u0007\u0002\u0000\u0000\u0234\u0235\u0007\u0005\u0000\u0000\u0235"+ + "\u0236\u0007\f\u0000\u0000\u0236\u0237\u0007\u0005\u0000\u0000\u0237\u0238"+ + "\u0007\u0002\u0000\u0000\u0238\u0239\u0001\u0000\u0000\u0000\u0239\u023a"+ + "\u0006\u000e\u0000\u0000\u023a-\u0001\u0000\u0000\u0000\u023b\u023c\u0007"+ + "\u0013\u0000\u0000\u023c\u023d\u0007\n\u0000\u0000\u023d\u023e\u0007\u0003"+ + "\u0000\u0000\u023e\u023f\u0007\u0006\u0000\u0000\u023f\u0240\u0007\u0003"+ + "\u0000\u0000\u0240\u0241\u0001\u0000\u0000\u0000\u0241\u0242\u0006\u000f"+ + "\u0000\u0000\u0242/\u0001\u0000\u0000\u0000\u0243\u0244\u0007\r\u0000"+ + "\u0000\u0244\u0245\u0007\u0007\u0000\u0000\u0245\u0246\u0007\u0007\u0000"+ + "\u0000\u0246\u0247\u0007\u0012\u0000\u0000\u0247\u0248\u0007\u0014\u0000"+ + "\u0000\u0248\u0249\u0007\b\u0000\u0000\u0249\u024a\u0001\u0000\u0000\u0000"+ + "\u024a\u024b\u0006\u0010\b\u0000\u024b1\u0001\u0000\u0000\u0000\u024c"+ + "\u024d\u0004\u0011\u0000\u0000\u024d\u024e\u0007\u0001\u0000\u0000\u024e"+ + "\u024f\u0007\t\u0000\u0000\u024f\u0250\u0007\r\u0000\u0000\u0250\u0251"+ + "\u0007\u0001\u0000\u0000\u0251\u0252\u0007\t\u0000\u0000\u0252\u0253\u0007"+ + "\u0003\u0000\u0000\u0253\u0254\u0007\u0002\u0000\u0000\u0254\u0255\u0007"+ + "\u0005\u0000\u0000\u0255\u0256\u0007\f\u0000\u0000\u0256\u0257\u0007\u0005"+ + "\u0000\u0000\u0257\u0258\u0007\u0002\u0000\u0000\u0258\u0259\u0001\u0000"+ + "\u0000\u0000\u0259\u025a\u0006\u0011\u0000\u0000\u025a3\u0001\u0000\u0000"+ + "\u0000\u025b\u025c\u0004\u0012\u0001\u0000\u025c\u025d\u0007\r\u0000\u0000"+ + "\u025d\u025e\u0007\u0007\u0000\u0000\u025e\u025f\u0007\u0007\u0000\u0000"+ + "\u025f\u0260\u0007\u0012\u0000\u0000\u0260\u0261\u0007\u0014\u0000\u0000"+ + "\u0261\u0262\u0007\b\u0000\u0000\u0262\u0263\u0005_\u0000\u0000\u0263"+ + "\u0264\u0005\u8001\uf414\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000"+ + "\u0265\u0266\u0006\u0012\t\u0000\u02665\u0001\u0000\u0000\u0000\u0267"+ + "\u0268\u0004\u0013\u0002\u0000\u0268\u0269\u0007\u0010\u0000\u0000\u0269"+ + "\u026a\u0007\u0003\u0000\u0000\u026a\u026b\u0007\u0005\u0000\u0000\u026b"+ + "\u026c\u0007\u0006\u0000\u0000\u026c\u026d\u0007\u0001\u0000\u0000\u026d"+ + "\u026e\u0007\u0004\u0000\u0000\u026e\u026f\u0007\u0002\u0000\u0000\u026f"+ + "\u0270\u0001\u0000\u0000\u0000\u0270\u0271\u0006\u0013\n\u0000\u02717"+ + "\u0001\u0000\u0000\u0000\u0272\u0273\u0004\u0014\u0003\u0000\u0273\u0274"+ + "\u0007\u000f\u0000\u0000\u0274\u0275\u0007\u0014\u0000\u0000\u0275\u0276"+ + "\u0007\r\u0000\u0000\u0276\u0277\u0007\r\u0000\u0000\u0277\u0278\u0001"+ + "\u0000\u0000\u0000\u0278\u0279\u0006\u0014\b\u0000\u02799\u0001\u0000"+ + "\u0000\u0000\u027a\u027b\u0004\u0015\u0004\u0000\u027b\u027c\u0007\r\u0000"+ + "\u0000\u027c\u027d\u0007\u0003\u0000\u0000\u027d\u027e\u0007\u000f\u0000"+ + "\u0000\u027e\u027f\u0007\u0005\u0000\u0000\u027f\u0280\u0001\u0000\u0000"+ + "\u0000\u0280\u0281\u0006\u0015\b\u0000\u0281;\u0001\u0000\u0000\u0000"+ + "\u0282\u0283\u0004\u0016\u0005\u0000\u0283\u0284\u0007\u0006\u0000\u0000"+ + "\u0284\u0285\u0007\u0001\u0000\u0000\u0285\u0286\u0007\u0011\u0000\u0000"+ + "\u0286\u0287\u0007\n\u0000\u0000\u0287\u0288\u0007\u0005\u0000\u0000\u0288"+ + "\u0289\u0001\u0000\u0000\u0000\u0289\u028a\u0006\u0016\b\u0000\u028a="+ + "\u0001\u0000\u0000\u0000\u028b\u028d\b\u0015\u0000\u0000\u028c\u028b\u0001"+ + "\u0000\u0000\u0000\u028d\u028e\u0001\u0000\u0000\u0000\u028e\u028c\u0001"+ + "\u0000\u0000\u0000\u028e\u028f\u0001\u0000\u0000\u0000\u028f\u0290\u0001"+ + "\u0000\u0000\u0000\u0290\u0291\u0006\u0017\u0000\u0000\u0291?\u0001\u0000"+ + "\u0000\u0000\u0292\u0293\u0005/\u0000\u0000\u0293\u0294\u0005/\u0000\u0000"+ + "\u0294\u0298\u0001\u0000\u0000\u0000\u0295\u0297\b\u0016\u0000\u0000\u0296"+ + "\u0295\u0001\u0000\u0000\u0000\u0297\u029a\u0001\u0000\u0000\u0000\u0298"+ + "\u0296\u0001\u0000\u0000\u0000\u0298\u0299\u0001\u0000\u0000\u0000\u0299"+ + "\u029c\u0001\u0000\u0000\u0000\u029a\u0298\u0001\u0000\u0000\u0000\u029b"+ + "\u029d\u0005\r\u0000\u0000\u029c\u029b\u0001\u0000\u0000\u0000\u029c\u029d"+ + "\u0001\u0000\u0000\u0000\u029d\u029f\u0001\u0000\u0000\u0000\u029e\u02a0"+ + "\u0005\n\u0000\u0000\u029f\u029e\u0001\u0000\u0000\u0000\u029f\u02a0\u0001"+ + "\u0000\u0000\u0000\u02a0\u02a1\u0001\u0000\u0000\u0000\u02a1\u02a2\u0006"+ + "\u0018\u000b\u0000\u02a2A\u0001\u0000\u0000\u0000\u02a3\u02a4\u0005/\u0000"+ + "\u0000\u02a4\u02a5\u0005*\u0000\u0000\u02a5\u02aa\u0001\u0000\u0000\u0000"+ + "\u02a6\u02a9\u0003B\u0019\u0000\u02a7\u02a9\t\u0000\u0000\u0000\u02a8"+ + "\u02a6\u0001\u0000\u0000\u0000\u02a8\u02a7\u0001\u0000\u0000\u0000\u02a9"+ + "\u02ac\u0001\u0000\u0000\u0000\u02aa\u02ab\u0001\u0000\u0000\u0000\u02aa"+ + "\u02a8\u0001\u0000\u0000\u0000\u02ab\u02ad\u0001\u0000\u0000\u0000\u02ac"+ + "\u02aa\u0001\u0000\u0000\u0000\u02ad\u02ae\u0005*\u0000\u0000\u02ae\u02af"+ + "\u0005/\u0000\u0000\u02af\u02b0\u0001\u0000\u0000\u0000\u02b0\u02b1\u0006"+ + "\u0019\u000b\u0000\u02b1C\u0001\u0000\u0000\u0000\u02b2\u02b4\u0007\u0017"+ + "\u0000\u0000\u02b3\u02b2\u0001\u0000\u0000\u0000\u02b4\u02b5\u0001\u0000"+ + "\u0000\u0000\u02b5\u02b3\u0001\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000"+ + "\u0000\u0000\u02b6\u02b7\u0001\u0000\u0000\u0000\u02b7\u02b8\u0006\u001a"+ + "\u000b\u0000\u02b8E\u0001\u0000\u0000\u0000\u02b9\u02ba\u0005|\u0000\u0000"+ + "\u02ba\u02bb\u0001\u0000\u0000\u0000\u02bb\u02bc\u0006\u001b\f\u0000\u02bc"+ + "G\u0001\u0000\u0000\u0000\u02bd\u02be\u0007\u0018\u0000\u0000\u02beI\u0001"+ + "\u0000\u0000\u0000\u02bf\u02c0\u0007\u0019\u0000\u0000\u02c0K\u0001\u0000"+ + "\u0000\u0000\u02c1\u02c2\u0005\\\u0000\u0000\u02c2\u02c3\u0007\u001a\u0000"+ + "\u0000\u02c3M\u0001\u0000\u0000\u0000\u02c4\u02c5\b\u001b\u0000\u0000"+ + "\u02c5O\u0001\u0000\u0000\u0000\u02c6\u02c8\u0007\u0003\u0000\u0000\u02c7"+ + "\u02c9\u0007\u001c\u0000\u0000\u02c8\u02c7\u0001\u0000\u0000\u0000\u02c8"+ + "\u02c9\u0001\u0000\u0000\u0000\u02c9\u02cb\u0001\u0000\u0000\u0000\u02ca"+ + "\u02cc\u0003H\u001c\u0000\u02cb\u02ca\u0001\u0000\u0000\u0000\u02cc\u02cd"+ + "\u0001\u0000\u0000\u0000\u02cd\u02cb\u0001\u0000\u0000\u0000\u02cd\u02ce"+ + "\u0001\u0000\u0000\u0000\u02ceQ\u0001\u0000\u0000\u0000\u02cf\u02d0\u0005"+ + "@\u0000\u0000\u02d0S\u0001\u0000\u0000\u0000\u02d1\u02d2\u0005`\u0000"+ + "\u0000\u02d2U\u0001\u0000\u0000\u0000\u02d3\u02d7\b\u001d\u0000\u0000"+ + "\u02d4\u02d5\u0005`\u0000\u0000\u02d5\u02d7\u0005`\u0000\u0000\u02d6\u02d3"+ + "\u0001\u0000\u0000\u0000\u02d6\u02d4\u0001\u0000\u0000\u0000\u02d7W\u0001"+ + "\u0000\u0000\u0000\u02d8\u02d9\u0005_\u0000\u0000\u02d9Y\u0001\u0000\u0000"+ + "\u0000\u02da\u02de\u0003J\u001d\u0000\u02db\u02de\u0003H\u001c\u0000\u02dc"+ + "\u02de\u0003X$\u0000\u02dd\u02da\u0001\u0000\u0000\u0000\u02dd\u02db\u0001"+ + "\u0000\u0000\u0000\u02dd\u02dc\u0001\u0000\u0000\u0000\u02de[\u0001\u0000"+ + "\u0000\u0000\u02df\u02e4\u0005\"\u0000\u0000\u02e0\u02e3\u0003L\u001e"+ + "\u0000\u02e1\u02e3\u0003N\u001f\u0000\u02e2\u02e0\u0001\u0000\u0000\u0000"+ + "\u02e2\u02e1\u0001\u0000\u0000\u0000\u02e3\u02e6\u0001\u0000\u0000\u0000"+ + "\u02e4\u02e2\u0001\u0000\u0000\u0000\u02e4\u02e5\u0001\u0000\u0000\u0000"+ + "\u02e5\u02e7\u0001\u0000\u0000\u0000\u02e6\u02e4\u0001\u0000\u0000\u0000"+ + "\u02e7\u02fd\u0005\"\u0000\u0000\u02e8\u02e9\u0005\"\u0000\u0000\u02e9"+ + "\u02ea\u0005\"\u0000\u0000\u02ea\u02eb\u0005\"\u0000\u0000\u02eb\u02ef"+ + "\u0001\u0000\u0000\u0000\u02ec\u02ee\b\u0016\u0000\u0000\u02ed\u02ec\u0001"+ + "\u0000\u0000\u0000\u02ee\u02f1\u0001\u0000\u0000\u0000\u02ef\u02f0\u0001"+ + "\u0000\u0000\u0000\u02ef\u02ed\u0001\u0000\u0000\u0000\u02f0\u02f2\u0001"+ + "\u0000\u0000\u0000\u02f1\u02ef\u0001\u0000\u0000\u0000\u02f2\u02f3\u0005"+ "\"\u0000\u0000\u02f3\u02f4\u0005\"\u0000\u0000\u02f4\u02f5\u0005\"\u0000"+ - "\u0000\u02f5\u02f6\u0005\"\u0000\u0000\u02f6\u02fa\u0001\u0000\u0000\u0000"+ - "\u02f7\u02f9\b\u0017\u0000\u0000\u02f8\u02f7\u0001\u0000\u0000\u0000\u02f9"+ - "\u02fc\u0001\u0000\u0000\u0000\u02fa\u02fb\u0001\u0000\u0000\u0000\u02fa"+ - "\u02f8\u0001\u0000\u0000\u0000\u02fb\u02fd\u0001\u0000\u0000\u0000\u02fc"+ - "\u02fa\u0001\u0000\u0000\u0000\u02fd\u02fe\u0005\"\u0000\u0000\u02fe\u02ff"+ - "\u0005\"\u0000\u0000\u02ff\u0300\u0005\"\u0000\u0000\u0300\u0302\u0001"+ - "\u0000\u0000\u0000\u0301\u0303\u0005\"\u0000\u0000\u0302\u0301\u0001\u0000"+ - "\u0000\u0000\u0302\u0303\u0001\u0000\u0000\u0000\u0303\u0305\u0001\u0000"+ - "\u0000\u0000\u0304\u0306\u0005\"\u0000\u0000\u0305\u0304\u0001\u0000\u0000"+ - "\u0000\u0305\u0306\u0001\u0000\u0000\u0000\u0306\u0308\u0001\u0000\u0000"+ - "\u0000\u0307\u02ea\u0001\u0000\u0000\u0000\u0307\u02f3\u0001\u0000\u0000"+ - "\u0000\u0308_\u0001\u0000\u0000\u0000\u0309\u030b\u0003J\u001d\u0000\u030a"+ - "\u0309\u0001\u0000\u0000\u0000\u030b\u030c\u0001\u0000\u0000\u0000\u030c"+ - "\u030a\u0001\u0000\u0000\u0000\u030c\u030d\u0001\u0000\u0000\u0000\u030d"+ - "a\u0001\u0000\u0000\u0000\u030e\u0310\u0003J\u001d\u0000\u030f\u030e\u0001"+ - "\u0000\u0000\u0000\u0310\u0311\u0001\u0000\u0000\u0000\u0311\u030f\u0001"+ - "\u0000\u0000\u0000\u0311\u0312\u0001\u0000\u0000\u0000\u0312\u0313\u0001"+ - "\u0000\u0000\u0000\u0313\u0317\u0003t2\u0000\u0314\u0316\u0003J\u001d"+ - "\u0000\u0315\u0314\u0001\u0000\u0000\u0000\u0316\u0319\u0001\u0000\u0000"+ - "\u0000\u0317\u0315\u0001\u0000\u0000\u0000\u0317\u0318\u0001\u0000\u0000"+ - "\u0000\u0318\u0339\u0001\u0000\u0000\u0000\u0319\u0317\u0001\u0000\u0000"+ - "\u0000\u031a\u031c\u0003t2\u0000\u031b\u031d\u0003J\u001d\u0000\u031c"+ - "\u031b\u0001\u0000\u0000\u0000\u031d\u031e\u0001\u0000\u0000\u0000\u031e"+ - "\u031c\u0001\u0000\u0000\u0000\u031e\u031f\u0001\u0000\u0000\u0000\u031f"+ - "\u0339\u0001\u0000\u0000\u0000\u0320\u0322\u0003J\u001d\u0000\u0321\u0320"+ - "\u0001\u0000\u0000\u0000\u0322\u0323\u0001\u0000\u0000\u0000\u0323\u0321"+ - "\u0001\u0000\u0000\u0000\u0323\u0324\u0001\u0000\u0000\u0000\u0324\u032c"+ - "\u0001\u0000\u0000\u0000\u0325\u0329\u0003t2\u0000\u0326\u0328\u0003J"+ - "\u001d\u0000\u0327\u0326\u0001\u0000\u0000\u0000\u0328\u032b\u0001\u0000"+ - "\u0000\u0000\u0329\u0327\u0001\u0000\u0000\u0000\u0329\u032a\u0001\u0000"+ - "\u0000\u0000\u032a\u032d\u0001\u0000\u0000\u0000\u032b\u0329\u0001\u0000"+ - "\u0000\u0000\u032c\u0325\u0001\u0000\u0000\u0000\u032c\u032d\u0001\u0000"+ - "\u0000\u0000\u032d\u032e\u0001\u0000\u0000\u0000\u032e\u032f\u0003R!\u0000"+ - "\u032f\u0339\u0001\u0000\u0000\u0000\u0330\u0332\u0003t2\u0000\u0331\u0333"+ - "\u0003J\u001d\u0000\u0332\u0331\u0001\u0000\u0000\u0000\u0333\u0334\u0001"+ - "\u0000\u0000\u0000\u0334\u0332\u0001\u0000\u0000\u0000\u0334\u0335\u0001"+ - "\u0000\u0000\u0000\u0335\u0336\u0001\u0000\u0000\u0000\u0336\u0337\u0003"+ - "R!\u0000\u0337\u0339\u0001\u0000\u0000\u0000\u0338\u030f\u0001\u0000\u0000"+ - "\u0000\u0338\u031a\u0001\u0000\u0000\u0000\u0338\u0321\u0001\u0000\u0000"+ - "\u0000\u0338\u0330\u0001\u0000\u0000\u0000\u0339c\u0001\u0000\u0000\u0000"+ - "\u033a\u033b\u0007\u001f\u0000\u0000\u033b\u033c\u0007 \u0000\u0000\u033c"+ - "e\u0001\u0000\u0000\u0000\u033d\u033e\u0007\f\u0000\u0000\u033e\u033f"+ - "\u0007\t\u0000\u0000\u033f\u0340\u0007\u0000\u0000\u0000\u0340g\u0001"+ - "\u0000\u0000\u0000\u0341\u0342\u0007\f\u0000\u0000\u0342\u0343\u0007\u0002"+ - "\u0000\u0000\u0343\u0344\u0007\u0004\u0000\u0000\u0344i\u0001\u0000\u0000"+ - "\u0000\u0345\u0346\u0005=\u0000\u0000\u0346k\u0001\u0000\u0000\u0000\u0347"+ - "\u0348\u0005:\u0000\u0000\u0348\u0349\u0005:\u0000\u0000\u0349m\u0001"+ - "\u0000\u0000\u0000\u034a\u034b\u0005:\u0000\u0000\u034bo\u0001\u0000\u0000"+ - "\u0000\u034c\u034d\u0005,\u0000\u0000\u034dq\u0001\u0000\u0000\u0000\u034e"+ - "\u034f\u0007\u0000\u0000\u0000\u034f\u0350\u0007\u0003\u0000\u0000\u0350"+ - "\u0351\u0007\u0002\u0000\u0000\u0351\u0352\u0007\u0004\u0000\u0000\u0352"+ - "s\u0001\u0000\u0000\u0000\u0353\u0354\u0005.\u0000\u0000\u0354u\u0001"+ - "\u0000\u0000\u0000\u0355\u0356\u0007\u000f\u0000\u0000\u0356\u0357\u0007"+ - "\f\u0000\u0000\u0357\u0358\u0007\r\u0000\u0000\u0358\u0359\u0007\u0002"+ - "\u0000\u0000\u0359\u035a\u0007\u0003\u0000\u0000\u035aw\u0001\u0000\u0000"+ - "\u0000\u035b\u035c\u0007\u000f\u0000\u0000\u035c\u035d\u0007\u0001\u0000"+ - "\u0000\u035d\u035e\u0007\u0006\u0000\u0000\u035e\u035f\u0007\u0002\u0000"+ - "\u0000\u035f\u0360\u0007\u0005\u0000\u0000\u0360y\u0001\u0000\u0000\u0000"+ - "\u0361\u0362\u0007\u0001\u0000\u0000\u0362\u0363\u0007\t\u0000\u0000\u0363"+ - "{\u0001\u0000\u0000\u0000\u0364\u0365\u0007\u0001\u0000\u0000\u0365\u0366"+ - "\u0007\u0002\u0000\u0000\u0366}\u0001\u0000\u0000\u0000\u0367\u0368\u0007"+ - "\r\u0000\u0000\u0368\u0369\u0007\f\u0000\u0000\u0369\u036a\u0007\u0002"+ - "\u0000\u0000\u036a\u036b\u0007\u0005\u0000\u0000\u036b\u007f\u0001\u0000"+ - "\u0000\u0000\u036c\u036d\u0007\r\u0000\u0000\u036d\u036e\u0007\u0001\u0000"+ - "\u0000\u036e\u036f\u0007\u0012\u0000\u0000\u036f\u0370\u0007\u0003\u0000"+ - "\u0000\u0370\u0081\u0001\u0000\u0000\u0000\u0371\u0372\u0005(\u0000\u0000"+ - "\u0372\u0083\u0001\u0000\u0000\u0000\u0373\u0374\u0007\t\u0000\u0000\u0374"+ - "\u0375\u0007\u0007\u0000\u0000\u0375\u0376\u0007\u0005\u0000\u0000\u0376"+ - "\u0085\u0001\u0000\u0000\u0000\u0377\u0378\u0007\t\u0000\u0000\u0378\u0379"+ - "\u0007\u0014\u0000\u0000\u0379\u037a\u0007\r\u0000\u0000\u037a\u037b\u0007"+ - "\r\u0000\u0000\u037b\u0087\u0001\u0000\u0000\u0000\u037c\u037d\u0007\t"+ - "\u0000\u0000\u037d\u037e\u0007\u0014\u0000\u0000\u037e\u037f\u0007\r\u0000"+ - "\u0000\u037f\u0380\u0007\r\u0000\u0000\u0380\u0381\u0007\u0002\u0000\u0000"+ - "\u0381\u0089\u0001\u0000\u0000\u0000\u0382\u0383\u0007\u0007\u0000\u0000"+ - "\u0383\u0384\u0007\u0006\u0000\u0000\u0384\u008b\u0001\u0000\u0000\u0000"+ - "\u0385\u0386\u0005?\u0000\u0000\u0386\u008d\u0001\u0000\u0000\u0000\u0387"+ - "\u0388\u0007\u0006\u0000\u0000\u0388\u0389\u0007\r\u0000\u0000\u0389\u038a"+ - "\u0007\u0001\u0000\u0000\u038a\u038b\u0007\u0012\u0000\u0000\u038b\u038c"+ - "\u0007\u0003\u0000\u0000\u038c\u008f\u0001\u0000\u0000\u0000\u038d\u038e"+ - "\u0005)\u0000\u0000\u038e\u0091\u0001\u0000\u0000\u0000\u038f\u0390\u0007"+ - "\u0005\u0000\u0000\u0390\u0391\u0007\u0006\u0000\u0000\u0391\u0392\u0007"+ - "\u0014\u0000\u0000\u0392\u0393\u0007\u0003\u0000\u0000\u0393\u0093\u0001"+ - "\u0000\u0000\u0000\u0394\u0395\u0005=\u0000\u0000\u0395\u0396\u0005=\u0000"+ - "\u0000\u0396\u0095\u0001\u0000\u0000\u0000\u0397\u0398\u0005=\u0000\u0000"+ - "\u0398\u0399\u0005~\u0000\u0000\u0399\u0097\u0001\u0000\u0000\u0000\u039a"+ - "\u039b\u0005!\u0000\u0000\u039b\u039c\u0005=\u0000\u0000\u039c\u0099\u0001"+ - "\u0000\u0000\u0000\u039d\u039e\u0005<\u0000\u0000\u039e\u009b\u0001\u0000"+ - "\u0000\u0000\u039f\u03a0\u0005<\u0000\u0000\u03a0\u03a1\u0005=\u0000\u0000"+ - "\u03a1\u009d\u0001\u0000\u0000\u0000\u03a2\u03a3\u0005>\u0000\u0000\u03a3"+ - "\u009f\u0001\u0000\u0000\u0000\u03a4\u03a5\u0005>\u0000\u0000\u03a5\u03a6"+ - "\u0005=\u0000\u0000\u03a6\u00a1\u0001\u0000\u0000\u0000\u03a7\u03a8\u0005"+ - "+\u0000\u0000\u03a8\u00a3\u0001\u0000\u0000\u0000\u03a9\u03aa\u0005-\u0000"+ - "\u0000\u03aa\u00a5\u0001\u0000\u0000\u0000\u03ab\u03ac\u0005*\u0000\u0000"+ - "\u03ac\u00a7\u0001\u0000\u0000\u0000\u03ad\u03ae\u0005/\u0000\u0000\u03ae"+ - "\u00a9\u0001\u0000\u0000\u0000\u03af\u03b0\u0005%\u0000\u0000\u03b0\u00ab"+ - "\u0001\u0000\u0000\u0000\u03b1\u03b2\u0005{\u0000\u0000\u03b2\u00ad\u0001"+ - "\u0000\u0000\u0000\u03b3\u03b4\u0005}\u0000\u0000\u03b4\u00af\u0001\u0000"+ - "\u0000\u0000\u03b5\u03b6\u0003.\u000f\u0000\u03b6\u03b7\u0001\u0000\u0000"+ - "\u0000\u03b7\u03b8\u0006P\r\u0000\u03b8\u00b1\u0001\u0000\u0000\u0000"+ - "\u03b9\u03bc\u0003\u008c>\u0000\u03ba\u03bd\u0003L\u001e\u0000\u03bb\u03bd"+ - "\u0003Z%\u0000\u03bc\u03ba\u0001\u0000\u0000\u0000\u03bc\u03bb\u0001\u0000"+ - "\u0000\u0000\u03bd\u03c1\u0001\u0000\u0000\u0000\u03be\u03c0\u0003\\&"+ - "\u0000\u03bf\u03be\u0001\u0000\u0000\u0000\u03c0\u03c3\u0001\u0000\u0000"+ - "\u0000\u03c1\u03bf\u0001\u0000\u0000\u0000\u03c1\u03c2\u0001\u0000\u0000"+ - "\u0000\u03c2\u03cb\u0001\u0000\u0000\u0000\u03c3\u03c1\u0001\u0000\u0000"+ - "\u0000\u03c4\u03c6\u0003\u008c>\u0000\u03c5\u03c7\u0003J\u001d\u0000\u03c6"+ - "\u03c5\u0001\u0000\u0000\u0000\u03c7\u03c8\u0001\u0000\u0000\u0000\u03c8"+ - "\u03c6\u0001\u0000\u0000\u0000\u03c8\u03c9\u0001\u0000\u0000\u0000\u03c9"+ - "\u03cb\u0001\u0000\u0000\u0000\u03ca\u03b9\u0001\u0000\u0000\u0000\u03ca"+ - "\u03c4\u0001\u0000\u0000\u0000\u03cb\u00b3\u0001\u0000\u0000\u0000\u03cc"+ - "\u03cd\u0005[\u0000\u0000\u03cd\u03ce\u0001\u0000\u0000\u0000\u03ce\u03cf"+ - "\u0006R\u0000\u0000\u03cf\u03d0\u0006R\u0000\u0000\u03d0\u00b5\u0001\u0000"+ - "\u0000\u0000\u03d1\u03d2\u0005]\u0000\u0000\u03d2\u03d3\u0001\u0000\u0000"+ - "\u0000\u03d3\u03d4\u0006S\f\u0000\u03d4\u03d5\u0006S\f\u0000\u03d5\u00b7"+ - "\u0001\u0000\u0000\u0000\u03d6\u03da\u0003L\u001e\u0000\u03d7\u03d9\u0003"+ - "\\&\u0000\u03d8\u03d7\u0001\u0000\u0000\u0000\u03d9\u03dc\u0001\u0000"+ - "\u0000\u0000\u03da\u03d8\u0001\u0000\u0000\u0000\u03da\u03db\u0001\u0000"+ - "\u0000\u0000\u03db\u03e7\u0001\u0000\u0000\u0000\u03dc\u03da\u0001\u0000"+ - "\u0000\u0000\u03dd\u03e0\u0003Z%\u0000\u03de\u03e0\u0003T\"\u0000\u03df"+ - "\u03dd\u0001\u0000\u0000\u0000\u03df\u03de\u0001\u0000\u0000\u0000\u03e0"+ - "\u03e2\u0001\u0000\u0000\u0000\u03e1\u03e3\u0003\\&\u0000\u03e2\u03e1"+ - "\u0001\u0000\u0000\u0000\u03e3\u03e4\u0001\u0000\u0000\u0000\u03e4\u03e2"+ - "\u0001\u0000\u0000\u0000\u03e4\u03e5\u0001\u0000\u0000\u0000\u03e5\u03e7"+ - "\u0001\u0000\u0000\u0000\u03e6\u03d6\u0001\u0000\u0000\u0000\u03e6\u03df"+ - "\u0001\u0000\u0000\u0000\u03e7\u00b9\u0001\u0000\u0000\u0000\u03e8\u03ea"+ - "\u0003V#\u0000\u03e9\u03eb\u0003X$\u0000\u03ea\u03e9\u0001\u0000\u0000"+ - "\u0000\u03eb\u03ec\u0001\u0000\u0000\u0000\u03ec\u03ea\u0001\u0000\u0000"+ - "\u0000\u03ec\u03ed\u0001\u0000\u0000\u0000\u03ed\u03ee\u0001\u0000\u0000"+ - "\u0000\u03ee\u03ef\u0003V#\u0000\u03ef\u00bb\u0001\u0000\u0000\u0000\u03f0"+ - "\u03f1\u0003\u00baU\u0000\u03f1\u00bd\u0001\u0000\u0000\u0000\u03f2\u03f3"+ - "\u0003B\u0019\u0000\u03f3\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006"+ - "W\u000b\u0000\u03f5\u00bf\u0001\u0000\u0000\u0000\u03f6\u03f7\u0003D\u001a"+ - "\u0000\u03f7\u03f8\u0001\u0000\u0000\u0000\u03f8\u03f9\u0006X\u000b\u0000"+ - "\u03f9\u00c1\u0001\u0000\u0000\u0000\u03fa\u03fb\u0003F\u001b\u0000\u03fb"+ - "\u03fc\u0001\u0000\u0000\u0000\u03fc\u03fd\u0006Y\u000b\u0000\u03fd\u00c3"+ - "\u0001\u0000\u0000\u0000\u03fe\u03ff\u0003\u00b4R\u0000\u03ff\u0400\u0001"+ - "\u0000\u0000\u0000\u0400\u0401\u0006Z\u000e\u0000\u0401\u0402\u0006Z\u000f"+ - "\u0000\u0402\u00c5\u0001\u0000\u0000\u0000\u0403\u0404\u0003H\u001c\u0000"+ - "\u0404\u0405\u0001\u0000\u0000\u0000\u0405\u0406\u0006[\u0010\u0000\u0406"+ - "\u0407\u0006[\f\u0000\u0407\u00c7\u0001\u0000\u0000\u0000\u0408\u0409"+ - "\u0003F\u001b\u0000\u0409\u040a\u0001\u0000\u0000\u0000\u040a\u040b\u0006"+ - "\\\u000b\u0000\u040b\u00c9\u0001\u0000\u0000\u0000\u040c\u040d\u0003B"+ - "\u0019\u0000\u040d\u040e\u0001\u0000\u0000\u0000\u040e\u040f\u0006]\u000b"+ - "\u0000\u040f\u00cb\u0001\u0000\u0000\u0000\u0410\u0411\u0003D\u001a\u0000"+ - "\u0411\u0412\u0001\u0000\u0000\u0000\u0412\u0413\u0006^\u000b\u0000\u0413"+ - "\u00cd\u0001\u0000\u0000\u0000\u0414\u0415\u0003H\u001c\u0000\u0415\u0416"+ - "\u0001\u0000\u0000\u0000\u0416\u0417\u0006_\u0010\u0000\u0417\u0418\u0006"+ - "_\f\u0000\u0418\u00cf\u0001\u0000\u0000\u0000\u0419\u041a\u0003\u00b4"+ - "R\u0000\u041a\u041b\u0001\u0000\u0000\u0000\u041b\u041c\u0006`\u000e\u0000"+ - "\u041c\u00d1\u0001\u0000\u0000\u0000\u041d\u041e\u0003\u00b6S\u0000\u041e"+ - "\u041f\u0001\u0000\u0000\u0000\u041f\u0420\u0006a\u0011\u0000\u0420\u00d3"+ - "\u0001\u0000\u0000\u0000\u0421\u0422\u0003n/\u0000\u0422\u0423\u0001\u0000"+ - "\u0000\u0000\u0423\u0424\u0006b\u0012\u0000\u0424\u00d5\u0001\u0000\u0000"+ - "\u0000\u0425\u0426\u0003p0\u0000\u0426\u0427\u0001\u0000\u0000\u0000\u0427"+ - "\u0428\u0006c\u0013\u0000\u0428\u00d7\u0001\u0000\u0000\u0000\u0429\u042a"+ - "\u0003j-\u0000\u042a\u042b\u0001\u0000\u0000\u0000\u042b\u042c\u0006d"+ - "\u0014\u0000\u042c\u00d9\u0001\u0000\u0000\u0000\u042d\u042e\u0007\u0010"+ - "\u0000\u0000\u042e\u042f\u0007\u0003\u0000\u0000\u042f\u0430\u0007\u0005"+ - "\u0000\u0000\u0430\u0431\u0007\f\u0000\u0000\u0431\u0432\u0007\u0000\u0000"+ - "\u0000\u0432\u0433\u0007\f\u0000\u0000\u0433\u0434\u0007\u0005\u0000\u0000"+ - "\u0434\u0435\u0007\f\u0000\u0000\u0435\u00db\u0001\u0000\u0000\u0000\u0436"+ - "\u043a\b!\u0000\u0000\u0437\u0438\u0005/\u0000\u0000\u0438\u043a\b\"\u0000"+ - "\u0000\u0439\u0436\u0001\u0000\u0000\u0000\u0439\u0437\u0001\u0000\u0000"+ - "\u0000\u043a\u00dd\u0001\u0000\u0000\u0000\u043b\u043d\u0003\u00dcf\u0000"+ - "\u043c\u043b\u0001\u0000\u0000\u0000\u043d\u043e\u0001\u0000\u0000\u0000"+ - "\u043e\u043c\u0001\u0000\u0000\u0000\u043e\u043f\u0001\u0000\u0000\u0000"+ - "\u043f\u00df\u0001\u0000\u0000\u0000\u0440\u0441\u0003\u00deg\u0000\u0441"+ - "\u0442\u0001\u0000\u0000\u0000\u0442\u0443\u0006h\u0015\u0000\u0443\u00e1"+ - "\u0001\u0000\u0000\u0000\u0444\u0445\u0003^\'\u0000\u0445\u0446\u0001"+ - "\u0000\u0000\u0000\u0446\u0447\u0006i\u0016\u0000\u0447\u00e3\u0001\u0000"+ - "\u0000\u0000\u0448\u0449\u0003B\u0019\u0000\u0449\u044a\u0001\u0000\u0000"+ - "\u0000\u044a\u044b\u0006j\u000b\u0000\u044b\u00e5\u0001\u0000\u0000\u0000"+ - "\u044c\u044d\u0003D\u001a\u0000\u044d\u044e\u0001\u0000\u0000\u0000\u044e"+ - "\u044f\u0006k\u000b\u0000\u044f\u00e7\u0001\u0000\u0000\u0000\u0450\u0451"+ - "\u0003F\u001b\u0000\u0451\u0452\u0001\u0000\u0000\u0000\u0452\u0453\u0006"+ - "l\u000b\u0000\u0453\u00e9\u0001\u0000\u0000\u0000\u0454\u0455\u0003H\u001c"+ - "\u0000\u0455\u0456\u0001\u0000\u0000\u0000\u0456\u0457\u0006m\u0010\u0000"+ - "\u0457\u0458\u0006m\f\u0000\u0458\u00eb\u0001\u0000\u0000\u0000\u0459"+ - "\u045a\u0003t2\u0000\u045a\u045b\u0001\u0000\u0000\u0000\u045b\u045c\u0006"+ - "n\u0017\u0000\u045c\u00ed\u0001\u0000\u0000\u0000\u045d\u045e\u0003p0"+ - "\u0000\u045e\u045f\u0001\u0000\u0000\u0000\u045f\u0460\u0006o\u0013\u0000"+ - "\u0460\u00ef\u0001\u0000\u0000\u0000\u0461\u0462\u0004p\b\u0000\u0462"+ - "\u0463\u0003\u008c>\u0000\u0463\u0464\u0001\u0000\u0000\u0000\u0464\u0465"+ - "\u0006p\u0018\u0000\u0465\u00f1\u0001\u0000\u0000\u0000\u0466\u0467\u0004"+ - "q\t\u0000\u0467\u0468\u0003\u00b2Q\u0000\u0468\u0469\u0001\u0000\u0000"+ - "\u0000\u0469\u046a\u0006q\u0019\u0000\u046a\u00f3\u0001\u0000\u0000\u0000"+ - "\u046b\u0470\u0003L\u001e\u0000\u046c\u0470\u0003J\u001d\u0000\u046d\u0470"+ - "\u0003Z%\u0000\u046e\u0470\u0003\u00a6K\u0000\u046f\u046b\u0001\u0000"+ - "\u0000\u0000\u046f\u046c\u0001\u0000\u0000\u0000\u046f\u046d\u0001\u0000"+ - "\u0000\u0000\u046f\u046e\u0001\u0000\u0000\u0000\u0470\u00f5\u0001\u0000"+ - "\u0000\u0000\u0471\u0474\u0003L\u001e\u0000\u0472\u0474\u0003\u00a6K\u0000"+ - "\u0473\u0471\u0001\u0000\u0000\u0000\u0473\u0472\u0001\u0000\u0000\u0000"+ - "\u0474\u0478\u0001\u0000\u0000\u0000\u0475\u0477\u0003\u00f4r\u0000\u0476"+ - "\u0475\u0001\u0000\u0000\u0000\u0477\u047a\u0001\u0000\u0000\u0000\u0478"+ - "\u0476\u0001\u0000\u0000\u0000\u0478\u0479\u0001\u0000\u0000\u0000\u0479"+ - "\u0485\u0001\u0000\u0000\u0000\u047a\u0478\u0001\u0000\u0000\u0000\u047b"+ - "\u047e\u0003Z%\u0000\u047c\u047e\u0003T\"\u0000\u047d\u047b\u0001\u0000"+ - "\u0000\u0000\u047d\u047c\u0001\u0000\u0000\u0000\u047e\u0480\u0001\u0000"+ - "\u0000\u0000\u047f\u0481\u0003\u00f4r\u0000\u0480\u047f\u0001\u0000\u0000"+ - "\u0000\u0481\u0482\u0001\u0000\u0000\u0000\u0482\u0480\u0001\u0000\u0000"+ - "\u0000\u0482\u0483\u0001\u0000\u0000\u0000\u0483\u0485\u0001\u0000\u0000"+ - "\u0000\u0484\u0473\u0001\u0000\u0000\u0000\u0484\u047d\u0001\u0000\u0000"+ - "\u0000\u0485\u00f7\u0001\u0000\u0000\u0000\u0486\u0489\u0003\u00f6s\u0000"+ - "\u0487\u0489\u0003\u00baU\u0000\u0488\u0486\u0001\u0000\u0000\u0000\u0488"+ - "\u0487\u0001\u0000\u0000\u0000\u0489\u048a\u0001\u0000\u0000\u0000\u048a"+ - "\u0488\u0001\u0000\u0000\u0000\u048a\u048b\u0001\u0000\u0000\u0000\u048b"+ - "\u00f9\u0001\u0000\u0000\u0000\u048c\u048d\u0003B\u0019\u0000\u048d\u048e"+ - "\u0001\u0000\u0000\u0000\u048e\u048f\u0006u\u000b\u0000\u048f\u00fb\u0001"+ - "\u0000\u0000\u0000\u0490\u0491\u0003D\u001a\u0000\u0491\u0492\u0001\u0000"+ - "\u0000\u0000\u0492\u0493\u0006v\u000b\u0000\u0493\u00fd\u0001\u0000\u0000"+ - "\u0000\u0494\u0495\u0003F\u001b\u0000\u0495\u0496\u0001\u0000\u0000\u0000"+ - "\u0496\u0497\u0006w\u000b\u0000\u0497\u00ff\u0001\u0000\u0000\u0000\u0498"+ - "\u0499\u0003H\u001c\u0000\u0499\u049a\u0001\u0000\u0000\u0000\u049a\u049b"+ - "\u0006x\u0010\u0000\u049b\u049c\u0006x\f\u0000\u049c\u0101\u0001\u0000"+ - "\u0000\u0000\u049d\u049e\u0003j-\u0000\u049e\u049f\u0001\u0000\u0000\u0000"+ - "\u049f\u04a0\u0006y\u0014\u0000\u04a0\u0103\u0001\u0000\u0000\u0000\u04a1"+ - "\u04a2\u0003p0\u0000\u04a2\u04a3\u0001\u0000\u0000\u0000\u04a3\u04a4\u0006"+ - "z\u0013\u0000\u04a4\u0105\u0001\u0000\u0000\u0000\u04a5\u04a6\u0003t2"+ - "\u0000\u04a6\u04a7\u0001\u0000\u0000\u0000\u04a7\u04a8\u0006{\u0017\u0000"+ - "\u04a8\u0107\u0001\u0000\u0000\u0000\u04a9\u04aa\u0004|\n\u0000\u04aa"+ - "\u04ab\u0003\u008c>\u0000\u04ab\u04ac\u0001\u0000\u0000\u0000\u04ac\u04ad"+ - "\u0006|\u0018\u0000\u04ad\u0109\u0001\u0000\u0000\u0000\u04ae\u04af\u0004"+ - "}\u000b\u0000\u04af\u04b0\u0003\u00b2Q\u0000\u04b0\u04b1\u0001\u0000\u0000"+ - "\u0000\u04b1\u04b2\u0006}\u0019\u0000\u04b2\u010b\u0001\u0000\u0000\u0000"+ - "\u04b3\u04b4\u0007\f\u0000\u0000\u04b4\u04b5\u0007\u0002\u0000\u0000\u04b5"+ - "\u010d\u0001\u0000\u0000\u0000\u04b6\u04b7\u0003\u00f8t\u0000\u04b7\u04b8"+ - "\u0001\u0000\u0000\u0000\u04b8\u04b9\u0006\u007f\u001a\u0000\u04b9\u010f"+ - "\u0001\u0000\u0000\u0000\u04ba\u04bb\u0003B\u0019\u0000\u04bb\u04bc\u0001"+ - "\u0000\u0000\u0000\u04bc\u04bd\u0006\u0080\u000b\u0000\u04bd\u0111\u0001"+ - "\u0000\u0000\u0000\u04be\u04bf\u0003D\u001a\u0000\u04bf\u04c0\u0001\u0000"+ - "\u0000\u0000\u04c0\u04c1\u0006\u0081\u000b\u0000\u04c1\u0113\u0001\u0000"+ - "\u0000\u0000\u04c2\u04c3\u0003F\u001b\u0000\u04c3\u04c4\u0001\u0000\u0000"+ - "\u0000\u04c4\u04c5\u0006\u0082\u000b\u0000\u04c5\u0115\u0001\u0000\u0000"+ - "\u0000\u04c6\u04c7\u0003H\u001c\u0000\u04c7\u04c8\u0001\u0000\u0000\u0000"+ - "\u04c8\u04c9\u0006\u0083\u0010\u0000\u04c9\u04ca\u0006\u0083\f\u0000\u04ca"+ - "\u0117\u0001\u0000\u0000\u0000\u04cb\u04cc\u0003\u00b4R\u0000\u04cc\u04cd"+ - "\u0001\u0000\u0000\u0000\u04cd\u04ce\u0006\u0084\u000e\u0000\u04ce\u04cf"+ - "\u0006\u0084\u001b\u0000\u04cf\u0119\u0001\u0000\u0000\u0000\u04d0\u04d1"+ - "\u0007\u0007\u0000\u0000\u04d1\u04d2\u0007\t\u0000\u0000\u04d2\u04d3\u0001"+ - "\u0000\u0000\u0000\u04d3\u04d4\u0006\u0085\u001c\u0000\u04d4\u011b\u0001"+ - "\u0000\u0000\u0000\u04d5\u04d6\u0007\u0013\u0000\u0000\u04d6\u04d7\u0007"+ - "\u0001\u0000\u0000\u04d7\u04d8\u0007\u0005\u0000\u0000\u04d8\u04d9\u0007"+ - "\n\u0000\u0000\u04d9\u04da\u0001\u0000\u0000\u0000\u04da\u04db\u0006\u0086"+ - "\u001c\u0000\u04db\u011d\u0001\u0000\u0000\u0000\u04dc\u04dd\b#\u0000"+ - "\u0000\u04dd\u011f\u0001\u0000\u0000\u0000\u04de\u04e0\u0003\u011e\u0087"+ - "\u0000\u04df\u04de\u0001\u0000\u0000\u0000\u04e0\u04e1\u0001\u0000\u0000"+ - "\u0000\u04e1\u04df\u0001\u0000\u0000\u0000\u04e1\u04e2\u0001\u0000\u0000"+ - "\u0000\u04e2\u04e3\u0001\u0000\u0000\u0000\u04e3\u04e4\u0003n/\u0000\u04e4"+ - "\u04e6\u0001\u0000\u0000\u0000\u04e5\u04df\u0001\u0000\u0000\u0000\u04e5"+ - "\u04e6\u0001\u0000\u0000\u0000\u04e6\u04e8\u0001\u0000\u0000\u0000\u04e7"+ - "\u04e9\u0003\u011e\u0087\u0000\u04e8\u04e7\u0001\u0000\u0000\u0000\u04e9"+ - "\u04ea\u0001\u0000\u0000\u0000\u04ea\u04e8\u0001\u0000\u0000\u0000\u04ea"+ - "\u04eb\u0001\u0000\u0000\u0000\u04eb\u0121\u0001\u0000\u0000\u0000\u04ec"+ - "\u04ed\u0003\u0120\u0088\u0000\u04ed\u04ee\u0001\u0000\u0000\u0000\u04ee"+ - "\u04ef\u0006\u0089\u001d\u0000\u04ef\u0123\u0001\u0000\u0000\u0000\u04f0"+ - "\u04f1\u0003B\u0019\u0000\u04f1\u04f2\u0001\u0000\u0000\u0000\u04f2\u04f3"+ - "\u0006\u008a\u000b\u0000\u04f3\u0125\u0001\u0000\u0000\u0000\u04f4\u04f5"+ - "\u0003D\u001a\u0000\u04f5\u04f6\u0001\u0000\u0000\u0000\u04f6\u04f7\u0006"+ - "\u008b\u000b\u0000\u04f7\u0127\u0001\u0000\u0000\u0000\u04f8\u04f9\u0003"+ - "F\u001b\u0000\u04f9\u04fa\u0001\u0000\u0000\u0000\u04fa\u04fb\u0006\u008c"+ - "\u000b\u0000\u04fb\u0129\u0001\u0000\u0000\u0000\u04fc\u04fd\u0003H\u001c"+ - "\u0000\u04fd\u04fe\u0001\u0000\u0000\u0000\u04fe\u04ff\u0006\u008d\u0010"+ - "\u0000\u04ff\u0500\u0006\u008d\f\u0000\u0500\u0501\u0006\u008d\f\u0000"+ - "\u0501\u012b\u0001\u0000\u0000\u0000\u0502\u0503\u0003j-\u0000\u0503\u0504"+ - "\u0001\u0000\u0000\u0000\u0504\u0505\u0006\u008e\u0014\u0000\u0505\u012d"+ - "\u0001\u0000\u0000\u0000\u0506\u0507\u0003p0\u0000\u0507\u0508\u0001\u0000"+ - "\u0000\u0000\u0508\u0509\u0006\u008f\u0013\u0000\u0509\u012f\u0001\u0000"+ - "\u0000\u0000\u050a\u050b\u0003t2\u0000\u050b\u050c\u0001\u0000\u0000\u0000"+ - "\u050c\u050d\u0006\u0090\u0017\u0000\u050d\u0131\u0001\u0000\u0000\u0000"+ - "\u050e\u050f\u0003\u011c\u0086\u0000\u050f\u0510\u0001\u0000\u0000\u0000"+ - "\u0510\u0511\u0006\u0091\u001e\u0000\u0511\u0133\u0001\u0000\u0000\u0000"+ - "\u0512\u0513\u0003\u00f8t\u0000\u0513\u0514\u0001\u0000\u0000\u0000\u0514"+ - "\u0515\u0006\u0092\u001a\u0000\u0515\u0135\u0001\u0000\u0000\u0000\u0516"+ - "\u0517\u0003\u00bcV\u0000\u0517\u0518\u0001\u0000\u0000\u0000\u0518\u0519"+ - "\u0006\u0093\u001f\u0000\u0519\u0137\u0001\u0000\u0000\u0000\u051a\u051b"+ - "\u0004\u0094\f\u0000\u051b\u051c\u0003\u008c>\u0000\u051c\u051d\u0001"+ - "\u0000\u0000\u0000\u051d\u051e\u0006\u0094\u0018\u0000\u051e\u0139\u0001"+ - "\u0000\u0000\u0000\u051f\u0520\u0004\u0095\r\u0000\u0520\u0521\u0003\u00b2"+ - "Q\u0000\u0521\u0522\u0001\u0000\u0000\u0000\u0522\u0523\u0006\u0095\u0019"+ - "\u0000\u0523\u013b\u0001\u0000\u0000\u0000\u0524\u0525\u0003B\u0019\u0000"+ - "\u0525\u0526\u0001\u0000\u0000\u0000\u0526\u0527\u0006\u0096\u000b\u0000"+ - "\u0527\u013d\u0001\u0000\u0000\u0000\u0528\u0529\u0003D\u001a\u0000\u0529"+ - "\u052a\u0001\u0000\u0000\u0000\u052a\u052b\u0006\u0097\u000b\u0000\u052b"+ - "\u013f\u0001\u0000\u0000\u0000\u052c\u052d\u0003F\u001b\u0000\u052d\u052e"+ - "\u0001\u0000\u0000\u0000\u052e\u052f\u0006\u0098\u000b\u0000\u052f\u0141"+ - "\u0001\u0000\u0000\u0000\u0530\u0531\u0003H\u001c\u0000\u0531\u0532\u0001"+ - "\u0000\u0000\u0000\u0532\u0533\u0006\u0099\u0010\u0000\u0533\u0534\u0006"+ - "\u0099\f\u0000\u0534\u0143\u0001\u0000\u0000\u0000\u0535\u0536\u0003t"+ - "2\u0000\u0536\u0537\u0001\u0000\u0000\u0000\u0537\u0538\u0006\u009a\u0017"+ - "\u0000\u0538\u0145\u0001\u0000\u0000\u0000\u0539\u053a\u0004\u009b\u000e"+ - "\u0000\u053a\u053b\u0003\u008c>\u0000\u053b\u053c\u0001\u0000\u0000\u0000"+ - "\u053c\u053d\u0006\u009b\u0018\u0000\u053d\u0147\u0001\u0000\u0000\u0000"+ - "\u053e\u053f\u0004\u009c\u000f\u0000\u053f\u0540\u0003\u00b2Q\u0000\u0540"+ - "\u0541\u0001\u0000\u0000\u0000\u0541\u0542\u0006\u009c\u0019\u0000\u0542"+ - "\u0149\u0001\u0000\u0000\u0000\u0543\u0544\u0003\u00bcV\u0000\u0544\u0545"+ - "\u0001\u0000\u0000\u0000\u0545\u0546\u0006\u009d\u001f\u0000\u0546\u014b"+ - "\u0001\u0000\u0000\u0000\u0547\u0548\u0003\u00b8T\u0000\u0548\u0549\u0001"+ - "\u0000\u0000\u0000\u0549\u054a\u0006\u009e \u0000\u054a\u014d\u0001\u0000"+ - "\u0000\u0000\u054b\u054c\u0003B\u0019\u0000\u054c\u054d\u0001\u0000\u0000"+ - "\u0000\u054d\u054e\u0006\u009f\u000b\u0000\u054e\u014f\u0001\u0000\u0000"+ - "\u0000\u054f\u0550\u0003D\u001a\u0000\u0550\u0551\u0001\u0000\u0000\u0000"+ - "\u0551\u0552\u0006\u00a0\u000b\u0000\u0552\u0151\u0001\u0000\u0000\u0000"+ - "\u0553\u0554\u0003F\u001b\u0000\u0554\u0555\u0001\u0000\u0000\u0000\u0555"+ - "\u0556\u0006\u00a1\u000b\u0000\u0556\u0153\u0001\u0000\u0000\u0000\u0557"+ - "\u0558\u0003H\u001c\u0000\u0558\u0559\u0001\u0000\u0000\u0000\u0559\u055a"+ - "\u0006\u00a2\u0010\u0000\u055a\u055b\u0006\u00a2\f\u0000\u055b\u0155\u0001"+ - "\u0000\u0000\u0000\u055c\u055d\u0007\u0001\u0000\u0000\u055d\u055e\u0007"+ - "\t\u0000\u0000\u055e\u055f\u0007\u000f\u0000\u0000\u055f\u0560\u0007\u0007"+ - "\u0000\u0000\u0560\u0157\u0001\u0000\u0000\u0000\u0561\u0562\u0003B\u0019"+ - "\u0000\u0562\u0563\u0001\u0000\u0000\u0000\u0563\u0564\u0006\u00a4\u000b"+ - "\u0000\u0564\u0159\u0001\u0000\u0000\u0000\u0565\u0566\u0003D\u001a\u0000"+ - "\u0566\u0567\u0001\u0000\u0000\u0000\u0567\u0568\u0006\u00a5\u000b\u0000"+ - "\u0568\u015b\u0001\u0000\u0000\u0000\u0569\u056a\u0003F\u001b\u0000\u056a"+ - "\u056b\u0001\u0000\u0000\u0000\u056b\u056c\u0006\u00a6\u000b\u0000\u056c"+ - "\u015d\u0001\u0000\u0000\u0000\u056d\u056e\u0003\u00b6S\u0000\u056e\u056f"+ - "\u0001\u0000\u0000\u0000\u056f\u0570\u0006\u00a7\u0011\u0000\u0570\u0571"+ - "\u0006\u00a7\f\u0000\u0571\u015f\u0001\u0000\u0000\u0000\u0572\u0573\u0003"+ - "n/\u0000\u0573\u0574\u0001\u0000\u0000\u0000\u0574\u0575\u0006\u00a8\u0012"+ - "\u0000\u0575\u0161\u0001\u0000\u0000\u0000\u0576\u057c\u0003T\"\u0000"+ - "\u0577\u057c\u0003J\u001d\u0000\u0578\u057c\u0003t2\u0000\u0579\u057c"+ - "\u0003L\u001e\u0000\u057a\u057c\u0003Z%\u0000\u057b\u0576\u0001\u0000"+ - "\u0000\u0000\u057b\u0577\u0001\u0000\u0000\u0000\u057b\u0578\u0001\u0000"+ - "\u0000\u0000\u057b\u0579\u0001\u0000\u0000\u0000\u057b\u057a\u0001\u0000"+ - "\u0000\u0000\u057c\u057d\u0001\u0000\u0000\u0000\u057d\u057b\u0001\u0000"+ - "\u0000\u0000\u057d\u057e\u0001\u0000\u0000\u0000\u057e\u0163\u0001\u0000"+ - "\u0000\u0000\u057f\u0580\u0003B\u0019\u0000\u0580\u0581\u0001\u0000\u0000"+ - "\u0000\u0581\u0582\u0006\u00aa\u000b\u0000\u0582\u0165\u0001\u0000\u0000"+ - "\u0000\u0583\u0584\u0003D\u001a\u0000\u0584\u0585\u0001\u0000\u0000\u0000"+ - "\u0585\u0586\u0006\u00ab\u000b\u0000\u0586\u0167\u0001\u0000\u0000\u0000"+ - "\u0587\u0588\u0003F\u001b\u0000\u0588\u0589\u0001\u0000\u0000\u0000\u0589"+ - "\u058a\u0006\u00ac\u000b\u0000\u058a\u0169\u0001\u0000\u0000\u0000\u058b"+ - "\u058c\u0003H\u001c\u0000\u058c\u058d\u0001\u0000\u0000\u0000\u058d\u058e"+ - "\u0006\u00ad\u0010\u0000\u058e\u058f\u0006\u00ad\f\u0000\u058f\u016b\u0001"+ - "\u0000\u0000\u0000\u0590\u0591\u0003n/\u0000\u0591\u0592\u0001\u0000\u0000"+ - "\u0000\u0592\u0593\u0006\u00ae\u0012\u0000\u0593\u016d\u0001\u0000\u0000"+ - "\u0000\u0594\u0595\u0003p0\u0000\u0595\u0596\u0001\u0000\u0000\u0000\u0596"+ - "\u0597\u0006\u00af\u0013\u0000\u0597\u016f\u0001\u0000\u0000\u0000\u0598"+ - "\u0599\u0003t2\u0000\u0599\u059a\u0001\u0000\u0000\u0000\u059a\u059b\u0006"+ - "\u00b0\u0017\u0000\u059b\u0171\u0001\u0000\u0000\u0000\u059c\u059d\u0003"+ - "\u011a\u0085\u0000\u059d\u059e\u0001\u0000\u0000\u0000\u059e\u059f\u0006"+ - "\u00b1!\u0000\u059f\u05a0\u0006\u00b1\"\u0000\u05a0\u0173\u0001\u0000"+ - "\u0000\u0000\u05a1\u05a2\u0003\u00deg\u0000\u05a2\u05a3\u0001\u0000\u0000"+ - "\u0000\u05a3\u05a4\u0006\u00b2\u0015\u0000\u05a4\u0175\u0001\u0000\u0000"+ - "\u0000\u05a5\u05a6\u0003^\'\u0000\u05a6\u05a7\u0001\u0000\u0000\u0000"+ - "\u05a7\u05a8\u0006\u00b3\u0016\u0000\u05a8\u0177\u0001\u0000\u0000\u0000"+ - "\u05a9\u05aa\u0003B\u0019\u0000\u05aa\u05ab\u0001\u0000\u0000\u0000\u05ab"+ - "\u05ac\u0006\u00b4\u000b\u0000\u05ac\u0179\u0001\u0000\u0000\u0000\u05ad"+ - "\u05ae\u0003D\u001a\u0000\u05ae\u05af\u0001\u0000\u0000\u0000\u05af\u05b0"+ - "\u0006\u00b5\u000b\u0000\u05b0\u017b\u0001\u0000\u0000\u0000\u05b1\u05b2"+ - "\u0003F\u001b\u0000\u05b2\u05b3\u0001\u0000\u0000\u0000\u05b3\u05b4\u0006"+ - "\u00b6\u000b\u0000\u05b4\u017d\u0001\u0000\u0000\u0000\u05b5\u05b6\u0003"+ - "H\u001c\u0000\u05b6\u05b7\u0001\u0000\u0000\u0000\u05b7\u05b8\u0006\u00b7"+ - "\u0010\u0000\u05b8\u05b9\u0006\u00b7\f\u0000\u05b9\u05ba\u0006\u00b7\f"+ - "\u0000\u05ba\u017f\u0001\u0000\u0000\u0000\u05bb\u05bc\u0003p0\u0000\u05bc"+ - "\u05bd\u0001\u0000\u0000\u0000\u05bd\u05be\u0006\u00b8\u0013\u0000\u05be"+ - "\u0181\u0001\u0000\u0000\u0000\u05bf\u05c0\u0003t2\u0000\u05c0\u05c1\u0001"+ - "\u0000\u0000\u0000\u05c1\u05c2\u0006\u00b9\u0017\u0000\u05c2\u0183\u0001"+ - "\u0000\u0000\u0000\u05c3\u05c4\u0003\u00f8t\u0000\u05c4\u05c5\u0001\u0000"+ - "\u0000\u0000\u05c5\u05c6\u0006\u00ba\u001a\u0000\u05c6\u0185\u0001\u0000"+ - "\u0000\u0000\u05c7\u05c8\u0003B\u0019\u0000\u05c8\u05c9\u0001\u0000\u0000"+ - "\u0000\u05c9\u05ca\u0006\u00bb\u000b\u0000\u05ca\u0187\u0001\u0000\u0000"+ - "\u0000\u05cb\u05cc\u0003D\u001a\u0000\u05cc\u05cd\u0001\u0000\u0000\u0000"+ - "\u05cd\u05ce\u0006\u00bc\u000b\u0000\u05ce\u0189\u0001\u0000\u0000\u0000"+ - "\u05cf\u05d0\u0003F\u001b\u0000\u05d0\u05d1\u0001\u0000\u0000\u0000\u05d1"+ - "\u05d2\u0006\u00bd\u000b\u0000\u05d2\u018b\u0001\u0000\u0000\u0000\u05d3"+ - "\u05d4\u0003H\u001c\u0000\u05d4\u05d5\u0001\u0000\u0000\u0000\u05d5\u05d6"+ - "\u0006\u00be\u0010\u0000\u05d6\u05d7\u0006\u00be\f\u0000\u05d7\u018d\u0001"+ - "\u0000\u0000\u0000\u05d8\u05d9\u00036\u0013\u0000\u05d9\u05da\u0001\u0000"+ - "\u0000\u0000\u05da\u05db\u0006\u00bf#\u0000\u05db\u018f\u0001\u0000\u0000"+ - "\u0000\u05dc\u05dd\u0003\u010c~\u0000\u05dd\u05de\u0001\u0000\u0000\u0000"+ - "\u05de\u05df\u0006\u00c0$\u0000\u05df\u0191\u0001\u0000\u0000\u0000\u05e0"+ - "\u05e1\u0003\u011a\u0085\u0000\u05e1\u05e2\u0001\u0000\u0000\u0000\u05e2"+ - "\u05e3\u0006\u00c1!\u0000\u05e3\u05e4\u0006\u00c1\f\u0000\u05e4\u05e5"+ - "\u0006\u00c1\u0000\u0000\u05e5\u0193\u0001\u0000\u0000\u0000\u05e6\u05e7"+ - "\u0007\u0014\u0000\u0000\u05e7\u05e8\u0007\u0002\u0000\u0000\u05e8\u05e9"+ - "\u0007\u0001\u0000\u0000\u05e9\u05ea\u0007\t\u0000\u0000\u05ea\u05eb\u0007"+ - "\u0011\u0000\u0000\u05eb\u05ec\u0001\u0000\u0000\u0000\u05ec\u05ed\u0006"+ - "\u00c2\f\u0000\u05ed\u05ee\u0006\u00c2\u0000\u0000\u05ee\u0195\u0001\u0000"+ - "\u0000\u0000\u05ef\u05f0\u0003\u00deg\u0000\u05f0\u05f1\u0001\u0000\u0000"+ - "\u0000\u05f1\u05f2\u0006\u00c3\u0015\u0000\u05f2\u0197\u0001\u0000\u0000"+ - "\u0000\u05f3\u05f4\u0003^\'\u0000\u05f4\u05f5\u0001\u0000\u0000\u0000"+ - "\u05f5\u05f6\u0006\u00c4\u0016\u0000\u05f6\u0199\u0001\u0000\u0000\u0000"+ - "\u05f7\u05f8\u0003n/\u0000\u05f8\u05f9\u0001\u0000\u0000\u0000\u05f9\u05fa"+ - "\u0006\u00c5\u0012\u0000\u05fa\u019b\u0001\u0000\u0000\u0000\u05fb\u05fc"+ - "\u0003\u00b8T\u0000\u05fc\u05fd\u0001\u0000\u0000\u0000\u05fd\u05fe\u0006"+ - "\u00c6 \u0000\u05fe\u019d\u0001\u0000\u0000\u0000\u05ff\u0600\u0003\u00bc"+ - "V\u0000\u0600\u0601\u0001\u0000\u0000\u0000\u0601\u0602\u0006\u00c7\u001f"+ - "\u0000\u0602\u019f\u0001\u0000\u0000\u0000\u0603\u0604\u0003B\u0019\u0000"+ - "\u0604\u0605\u0001\u0000\u0000\u0000\u0605\u0606\u0006\u00c8\u000b\u0000"+ - "\u0606\u01a1\u0001\u0000\u0000\u0000\u0607\u0608\u0003D\u001a\u0000\u0608"+ - "\u0609\u0001\u0000\u0000\u0000\u0609\u060a\u0006\u00c9\u000b\u0000\u060a"+ - "\u01a3\u0001\u0000\u0000\u0000\u060b\u060c\u0003F\u001b\u0000\u060c\u060d"+ - "\u0001\u0000\u0000\u0000\u060d\u060e\u0006\u00ca\u000b\u0000\u060e\u01a5"+ - "\u0001\u0000\u0000\u0000\u060f\u0610\u0003H\u001c\u0000\u0610\u0611\u0001"+ - "\u0000\u0000\u0000\u0611\u0612\u0006\u00cb\u0010\u0000\u0612\u0613\u0006"+ - "\u00cb\f\u0000\u0613\u01a7\u0001\u0000\u0000\u0000\u0614\u0615\u0003\u00de"+ - "g\u0000\u0615\u0616\u0001\u0000\u0000\u0000\u0616\u0617\u0006\u00cc\u0015"+ - "\u0000\u0617\u0618\u0006\u00cc\f\u0000\u0618\u0619\u0006\u00cc%\u0000"+ - "\u0619\u01a9\u0001\u0000\u0000\u0000\u061a\u061b\u0003^\'\u0000\u061b"+ - "\u061c\u0001\u0000\u0000\u0000\u061c\u061d\u0006\u00cd\u0016\u0000\u061d"+ - "\u061e\u0006\u00cd\f\u0000\u061e\u061f\u0006\u00cd%\u0000\u061f\u01ab"+ - "\u0001\u0000\u0000\u0000\u0620\u0621\u0003B\u0019\u0000\u0621\u0622\u0001"+ - "\u0000\u0000\u0000\u0622\u0623\u0006\u00ce\u000b\u0000\u0623\u01ad\u0001"+ - "\u0000\u0000\u0000\u0624\u0625\u0003D\u001a\u0000\u0625\u0626\u0001\u0000"+ - "\u0000\u0000\u0626\u0627\u0006\u00cf\u000b\u0000\u0627\u01af\u0001\u0000"+ - "\u0000\u0000\u0628\u0629\u0003F\u001b\u0000\u0629\u062a\u0001\u0000\u0000"+ - "\u0000\u062a\u062b\u0006\u00d0\u000b\u0000\u062b\u01b1\u0001\u0000\u0000"+ - "\u0000\u062c\u062d\u0003n/\u0000\u062d\u062e\u0001\u0000\u0000\u0000\u062e"+ - "\u062f\u0006\u00d1\u0012\u0000\u062f\u0630\u0006\u00d1\f\u0000\u0630\u0631"+ - "\u0006\u00d1\t\u0000\u0631\u01b3\u0001\u0000\u0000\u0000\u0632\u0633\u0003"+ - "p0\u0000\u0633\u0634\u0001\u0000\u0000\u0000\u0634\u0635\u0006\u00d2\u0013"+ - "\u0000\u0635\u0636\u0006\u00d2\f\u0000\u0636\u0637\u0006\u00d2\t\u0000"+ - "\u0637\u01b5\u0001\u0000\u0000\u0000\u0638\u0639\u0003B\u0019\u0000\u0639"+ - "\u063a\u0001\u0000\u0000\u0000\u063a\u063b\u0006\u00d3\u000b\u0000\u063b"+ - "\u01b7\u0001\u0000\u0000\u0000\u063c\u063d\u0003D\u001a\u0000\u063d\u063e"+ - "\u0001\u0000\u0000\u0000\u063e\u063f\u0006\u00d4\u000b\u0000\u063f\u01b9"+ - "\u0001\u0000\u0000\u0000\u0640\u0641\u0003F\u001b\u0000\u0641\u0642\u0001"+ - "\u0000\u0000\u0000\u0642\u0643\u0006\u00d5\u000b\u0000\u0643\u01bb\u0001"+ - "\u0000\u0000\u0000\u0644\u0645\u0003\u00bcV\u0000\u0645\u0646\u0001\u0000"+ - "\u0000\u0000\u0646\u0647\u0006\u00d6\f\u0000\u0647\u0648\u0006\u00d6\u0000"+ - "\u0000\u0648\u0649\u0006\u00d6\u001f\u0000\u0649\u01bd\u0001\u0000\u0000"+ - "\u0000\u064a\u064b\u0003\u00b8T\u0000\u064b\u064c\u0001\u0000\u0000\u0000"+ - "\u064c\u064d\u0006\u00d7\f\u0000\u064d\u064e\u0006\u00d7\u0000\u0000\u064e"+ - "\u064f\u0006\u00d7 \u0000\u064f\u01bf\u0001\u0000\u0000\u0000\u0650\u0651"+ - "\u0003d*\u0000\u0651\u0652\u0001\u0000\u0000\u0000\u0652\u0653\u0006\u00d8"+ - "\f\u0000\u0653\u0654\u0006\u00d8\u0000\u0000\u0654\u0655\u0006\u00d8&"+ - "\u0000\u0655\u01c1\u0001\u0000\u0000\u0000\u0656\u0657\u0003H\u001c\u0000"+ - "\u0657\u0658\u0001\u0000\u0000\u0000\u0658\u0659\u0006\u00d9\u0010\u0000"+ - "\u0659\u065a\u0006\u00d9\f\u0000\u065a\u01c3\u0001\u0000\u0000\u0000B"+ + "\u0000\u02f5\u02f7\u0001\u0000\u0000\u0000\u02f6\u02f8\u0005\"\u0000\u0000"+ + "\u02f7\u02f6\u0001\u0000\u0000\u0000\u02f7\u02f8\u0001\u0000\u0000\u0000"+ + "\u02f8\u02fa\u0001\u0000\u0000\u0000\u02f9\u02fb\u0005\"\u0000\u0000\u02fa"+ + "\u02f9\u0001\u0000\u0000\u0000\u02fa\u02fb\u0001\u0000\u0000\u0000\u02fb"+ + "\u02fd\u0001\u0000\u0000\u0000\u02fc\u02df\u0001\u0000\u0000\u0000\u02fc"+ + "\u02e8\u0001\u0000\u0000\u0000\u02fd]\u0001\u0000\u0000\u0000\u02fe\u0300"+ + "\u0003H\u001c\u0000\u02ff\u02fe\u0001\u0000\u0000\u0000\u0300\u0301\u0001"+ + "\u0000\u0000\u0000\u0301\u02ff\u0001\u0000\u0000\u0000\u0301\u0302\u0001"+ + "\u0000\u0000\u0000\u0302_\u0001\u0000\u0000\u0000\u0303\u0305\u0003H\u001c"+ + "\u0000\u0304\u0303\u0001\u0000\u0000\u0000\u0305\u0306\u0001\u0000\u0000"+ + "\u0000\u0306\u0304\u0001\u0000\u0000\u0000\u0306\u0307\u0001\u0000\u0000"+ + "\u0000\u0307\u0308\u0001\u0000\u0000\u0000\u0308\u030c\u0003r1\u0000\u0309"+ + "\u030b\u0003H\u001c\u0000\u030a\u0309\u0001\u0000\u0000\u0000\u030b\u030e"+ + "\u0001\u0000\u0000\u0000\u030c\u030a\u0001\u0000\u0000\u0000\u030c\u030d"+ + "\u0001\u0000\u0000\u0000\u030d\u032e\u0001\u0000\u0000\u0000\u030e\u030c"+ + "\u0001\u0000\u0000\u0000\u030f\u0311\u0003r1\u0000\u0310\u0312\u0003H"+ + "\u001c\u0000\u0311\u0310\u0001\u0000\u0000\u0000\u0312\u0313\u0001\u0000"+ + "\u0000\u0000\u0313\u0311\u0001\u0000\u0000\u0000\u0313\u0314\u0001\u0000"+ + "\u0000\u0000\u0314\u032e\u0001\u0000\u0000\u0000\u0315\u0317\u0003H\u001c"+ + "\u0000\u0316\u0315\u0001\u0000\u0000\u0000\u0317\u0318\u0001\u0000\u0000"+ + "\u0000\u0318\u0316\u0001\u0000\u0000\u0000\u0318\u0319\u0001\u0000\u0000"+ + "\u0000\u0319\u0321\u0001\u0000\u0000\u0000\u031a\u031e\u0003r1\u0000\u031b"+ + "\u031d\u0003H\u001c\u0000\u031c\u031b\u0001\u0000\u0000\u0000\u031d\u0320"+ + "\u0001\u0000\u0000\u0000\u031e\u031c\u0001\u0000\u0000\u0000\u031e\u031f"+ + "\u0001\u0000\u0000\u0000\u031f\u0322\u0001\u0000\u0000\u0000\u0320\u031e"+ + "\u0001\u0000\u0000\u0000\u0321\u031a\u0001\u0000\u0000\u0000\u0321\u0322"+ + "\u0001\u0000\u0000\u0000\u0322\u0323\u0001\u0000\u0000\u0000\u0323\u0324"+ + "\u0003P \u0000\u0324\u032e\u0001\u0000\u0000\u0000\u0325\u0327\u0003r"+ + "1\u0000\u0326\u0328\u0003H\u001c\u0000\u0327\u0326\u0001\u0000\u0000\u0000"+ + "\u0328\u0329\u0001\u0000\u0000\u0000\u0329\u0327\u0001\u0000\u0000\u0000"+ + "\u0329\u032a\u0001\u0000\u0000\u0000\u032a\u032b\u0001\u0000\u0000\u0000"+ + "\u032b\u032c\u0003P \u0000\u032c\u032e\u0001\u0000\u0000\u0000\u032d\u0304"+ + "\u0001\u0000\u0000\u0000\u032d\u030f\u0001\u0000\u0000\u0000\u032d\u0316"+ + "\u0001\u0000\u0000\u0000\u032d\u0325\u0001\u0000\u0000\u0000\u032ea\u0001"+ + "\u0000\u0000\u0000\u032f\u0330\u0007\u001e\u0000\u0000\u0330\u0331\u0007"+ + "\u001f\u0000\u0000\u0331c\u0001\u0000\u0000\u0000\u0332\u0333\u0007\f"+ + "\u0000\u0000\u0333\u0334\u0007\t\u0000\u0000\u0334\u0335\u0007\u0000\u0000"+ + "\u0000\u0335e\u0001\u0000\u0000\u0000\u0336\u0337\u0007\f\u0000\u0000"+ + "\u0337\u0338\u0007\u0002\u0000\u0000\u0338\u0339\u0007\u0004\u0000\u0000"+ + "\u0339g\u0001\u0000\u0000\u0000\u033a\u033b\u0005=\u0000\u0000\u033bi"+ + "\u0001\u0000\u0000\u0000\u033c\u033d\u0005:\u0000\u0000\u033d\u033e\u0005"+ + ":\u0000\u0000\u033ek\u0001\u0000\u0000\u0000\u033f\u0340\u0005:\u0000"+ + "\u0000\u0340m\u0001\u0000\u0000\u0000\u0341\u0342\u0005,\u0000\u0000\u0342"+ + "o\u0001\u0000\u0000\u0000\u0343\u0344\u0007\u0000\u0000\u0000\u0344\u0345"+ + "\u0007\u0003\u0000\u0000\u0345\u0346\u0007\u0002\u0000\u0000\u0346\u0347"+ + "\u0007\u0004\u0000\u0000\u0347q\u0001\u0000\u0000\u0000\u0348\u0349\u0005"+ + ".\u0000\u0000\u0349s\u0001\u0000\u0000\u0000\u034a\u034b\u0007\u000f\u0000"+ + "\u0000\u034b\u034c\u0007\f\u0000\u0000\u034c\u034d\u0007\r\u0000\u0000"+ + "\u034d\u034e\u0007\u0002\u0000\u0000\u034e\u034f\u0007\u0003\u0000\u0000"+ + "\u034fu\u0001\u0000\u0000\u0000\u0350\u0351\u0007\u000f\u0000\u0000\u0351"+ + "\u0352\u0007\u0001\u0000\u0000\u0352\u0353\u0007\u0006\u0000\u0000\u0353"+ + "\u0354\u0007\u0002\u0000\u0000\u0354\u0355\u0007\u0005\u0000\u0000\u0355"+ + "w\u0001\u0000\u0000\u0000\u0356\u0357\u0007\u0001\u0000\u0000\u0357\u0358"+ + "\u0007\t\u0000\u0000\u0358y\u0001\u0000\u0000\u0000\u0359\u035a\u0007"+ + "\u0001\u0000\u0000\u035a\u035b\u0007\u0002\u0000\u0000\u035b{\u0001\u0000"+ + "\u0000\u0000\u035c\u035d\u0007\r\u0000\u0000\u035d\u035e\u0007\f\u0000"+ + "\u0000\u035e\u035f\u0007\u0002\u0000\u0000\u035f\u0360\u0007\u0005\u0000"+ + "\u0000\u0360}\u0001\u0000\u0000\u0000\u0361\u0362\u0007\r\u0000\u0000"+ + "\u0362\u0363\u0007\u0001\u0000\u0000\u0363\u0364\u0007\u0012\u0000\u0000"+ + "\u0364\u0365\u0007\u0003\u0000\u0000\u0365\u007f\u0001\u0000\u0000\u0000"+ + "\u0366\u0367\u0005(\u0000\u0000\u0367\u0081\u0001\u0000\u0000\u0000\u0368"+ + "\u0369\u0007\t\u0000\u0000\u0369\u036a\u0007\u0007\u0000\u0000\u036a\u036b"+ + "\u0007\u0005\u0000\u0000\u036b\u0083\u0001\u0000\u0000\u0000\u036c\u036d"+ + "\u0007\t\u0000\u0000\u036d\u036e\u0007\u0014\u0000\u0000\u036e\u036f\u0007"+ + "\r\u0000\u0000\u036f\u0370\u0007\r\u0000\u0000\u0370\u0085\u0001\u0000"+ + "\u0000\u0000\u0371\u0372\u0007\t\u0000\u0000\u0372\u0373\u0007\u0014\u0000"+ + "\u0000\u0373\u0374\u0007\r\u0000\u0000\u0374\u0375\u0007\r\u0000\u0000"+ + "\u0375\u0376\u0007\u0002\u0000\u0000\u0376\u0087\u0001\u0000\u0000\u0000"+ + "\u0377\u0378\u0007\u0007\u0000\u0000\u0378\u0379\u0007\u0006\u0000\u0000"+ + "\u0379\u0089\u0001\u0000\u0000\u0000\u037a\u037b\u0005?\u0000\u0000\u037b"+ + "\u008b\u0001\u0000\u0000\u0000\u037c\u037d\u0007\u0006\u0000\u0000\u037d"+ + "\u037e\u0007\r\u0000\u0000\u037e\u037f\u0007\u0001\u0000\u0000\u037f\u0380"+ + "\u0007\u0012\u0000\u0000\u0380\u0381\u0007\u0003\u0000\u0000\u0381\u008d"+ + "\u0001\u0000\u0000\u0000\u0382\u0383\u0005)\u0000\u0000\u0383\u008f\u0001"+ + "\u0000\u0000\u0000\u0384\u0385\u0007\u0005\u0000\u0000\u0385\u0386\u0007"+ + "\u0006\u0000\u0000\u0386\u0387\u0007\u0014\u0000\u0000\u0387\u0388\u0007"+ + "\u0003\u0000\u0000\u0388\u0091\u0001\u0000\u0000\u0000\u0389\u038a\u0005"+ + "=\u0000\u0000\u038a\u038b\u0005=\u0000\u0000\u038b\u0093\u0001\u0000\u0000"+ + "\u0000\u038c\u038d\u0005=\u0000\u0000\u038d\u038e\u0005~\u0000\u0000\u038e"+ + "\u0095\u0001\u0000\u0000\u0000\u038f\u0390\u0005!\u0000\u0000\u0390\u0391"+ + "\u0005=\u0000\u0000\u0391\u0097\u0001\u0000\u0000\u0000\u0392\u0393\u0005"+ + "<\u0000\u0000\u0393\u0099\u0001\u0000\u0000\u0000\u0394\u0395\u0005<\u0000"+ + "\u0000\u0395\u0396\u0005=\u0000\u0000\u0396\u009b\u0001\u0000\u0000\u0000"+ + "\u0397\u0398\u0005>\u0000\u0000\u0398\u009d\u0001\u0000\u0000\u0000\u0399"+ + "\u039a\u0005>\u0000\u0000\u039a\u039b\u0005=\u0000\u0000\u039b\u009f\u0001"+ + "\u0000\u0000\u0000\u039c\u039d\u0005+\u0000\u0000\u039d\u00a1\u0001\u0000"+ + "\u0000\u0000\u039e\u039f\u0005-\u0000\u0000\u039f\u00a3\u0001\u0000\u0000"+ + "\u0000\u03a0\u03a1\u0005*\u0000\u0000\u03a1\u00a5\u0001\u0000\u0000\u0000"+ + "\u03a2\u03a3\u0005/\u0000\u0000\u03a3\u00a7\u0001\u0000\u0000\u0000\u03a4"+ + "\u03a5\u0005%\u0000\u0000\u03a5\u00a9\u0001\u0000\u0000\u0000\u03a6\u03a7"+ + "\u0005{\u0000\u0000\u03a7\u00ab\u0001\u0000\u0000\u0000\u03a8\u03a9\u0005"+ + "}\u0000\u0000\u03a9\u00ad\u0001\u0000\u0000\u0000\u03aa\u03ab\u0003.\u000f"+ + "\u0000\u03ab\u03ac\u0001\u0000\u0000\u0000\u03ac\u03ad\u0006O\r\u0000"+ + "\u03ad\u00af\u0001\u0000\u0000\u0000\u03ae\u03b1\u0003\u008a=\u0000\u03af"+ + "\u03b2\u0003J\u001d\u0000\u03b0\u03b2\u0003X$\u0000\u03b1\u03af\u0001"+ + "\u0000\u0000\u0000\u03b1\u03b0\u0001\u0000\u0000\u0000\u03b2\u03b6\u0001"+ + "\u0000\u0000\u0000\u03b3\u03b5\u0003Z%\u0000\u03b4\u03b3\u0001\u0000\u0000"+ + "\u0000\u03b5\u03b8\u0001\u0000\u0000\u0000\u03b6\u03b4\u0001\u0000\u0000"+ + "\u0000\u03b6\u03b7\u0001\u0000\u0000\u0000\u03b7\u03c0\u0001\u0000\u0000"+ + "\u0000\u03b8\u03b6\u0001\u0000\u0000\u0000\u03b9\u03bb\u0003\u008a=\u0000"+ + "\u03ba\u03bc\u0003H\u001c\u0000\u03bb\u03ba\u0001\u0000\u0000\u0000\u03bc"+ + "\u03bd\u0001\u0000\u0000\u0000\u03bd\u03bb\u0001\u0000\u0000\u0000\u03bd"+ + "\u03be\u0001\u0000\u0000\u0000\u03be\u03c0\u0001\u0000\u0000\u0000\u03bf"+ + "\u03ae\u0001\u0000\u0000\u0000\u03bf\u03b9\u0001\u0000\u0000\u0000\u03c0"+ + "\u00b1\u0001\u0000\u0000\u0000\u03c1\u03c2\u0005[\u0000\u0000\u03c2\u03c3"+ + "\u0001\u0000\u0000\u0000\u03c3\u03c4\u0006Q\u0000\u0000\u03c4\u03c5\u0006"+ + "Q\u0000\u0000\u03c5\u00b3\u0001\u0000\u0000\u0000\u03c6\u03c7\u0005]\u0000"+ + "\u0000\u03c7\u03c8\u0001\u0000\u0000\u0000\u03c8\u03c9\u0006R\f\u0000"+ + "\u03c9\u03ca\u0006R\f\u0000\u03ca\u00b5\u0001\u0000\u0000\u0000\u03cb"+ + "\u03cf\u0003J\u001d\u0000\u03cc\u03ce\u0003Z%\u0000\u03cd\u03cc\u0001"+ + "\u0000\u0000\u0000\u03ce\u03d1\u0001\u0000\u0000\u0000\u03cf\u03cd\u0001"+ + "\u0000\u0000\u0000\u03cf\u03d0\u0001\u0000\u0000\u0000\u03d0\u03dc\u0001"+ + "\u0000\u0000\u0000\u03d1\u03cf\u0001\u0000\u0000\u0000\u03d2\u03d5\u0003"+ + "X$\u0000\u03d3\u03d5\u0003R!\u0000\u03d4\u03d2\u0001\u0000\u0000\u0000"+ + "\u03d4\u03d3\u0001\u0000\u0000\u0000\u03d5\u03d7\u0001\u0000\u0000\u0000"+ + "\u03d6\u03d8\u0003Z%\u0000\u03d7\u03d6\u0001\u0000\u0000\u0000\u03d8\u03d9"+ + "\u0001\u0000\u0000\u0000\u03d9\u03d7\u0001\u0000\u0000\u0000\u03d9\u03da"+ + "\u0001\u0000\u0000\u0000\u03da\u03dc\u0001\u0000\u0000\u0000\u03db\u03cb"+ + "\u0001\u0000\u0000\u0000\u03db\u03d4\u0001\u0000\u0000\u0000\u03dc\u00b7"+ + "\u0001\u0000\u0000\u0000\u03dd\u03df\u0003T\"\u0000\u03de\u03e0\u0003"+ + "V#\u0000\u03df\u03de\u0001\u0000\u0000\u0000\u03e0\u03e1\u0001\u0000\u0000"+ + "\u0000\u03e1\u03df\u0001\u0000\u0000\u0000\u03e1\u03e2\u0001\u0000\u0000"+ + "\u0000\u03e2\u03e3\u0001\u0000\u0000\u0000\u03e3\u03e4\u0003T\"\u0000"+ + "\u03e4\u00b9\u0001\u0000\u0000\u0000\u03e5\u03e6\u0003\u00b8T\u0000\u03e6"+ + "\u00bb\u0001\u0000\u0000\u0000\u03e7\u03e8\u0003@\u0018\u0000\u03e8\u03e9"+ + "\u0001\u0000\u0000\u0000\u03e9\u03ea\u0006V\u000b\u0000\u03ea\u00bd\u0001"+ + "\u0000\u0000\u0000\u03eb\u03ec\u0003B\u0019\u0000\u03ec\u03ed\u0001\u0000"+ + "\u0000\u0000\u03ed\u03ee\u0006W\u000b\u0000\u03ee\u00bf\u0001\u0000\u0000"+ + "\u0000\u03ef\u03f0\u0003D\u001a\u0000\u03f0\u03f1\u0001\u0000\u0000\u0000"+ + "\u03f1\u03f2\u0006X\u000b\u0000\u03f2\u00c1\u0001\u0000\u0000\u0000\u03f3"+ + "\u03f4\u0003\u00b2Q\u0000\u03f4\u03f5\u0001\u0000\u0000\u0000\u03f5\u03f6"+ + "\u0006Y\u000e\u0000\u03f6\u03f7\u0006Y\u000f\u0000\u03f7\u00c3\u0001\u0000"+ + "\u0000\u0000\u03f8\u03f9\u0003F\u001b\u0000\u03f9\u03fa\u0001\u0000\u0000"+ + "\u0000\u03fa\u03fb\u0006Z\u0010\u0000\u03fb\u03fc\u0006Z\f\u0000\u03fc"+ + "\u00c5\u0001\u0000\u0000\u0000\u03fd\u03fe\u0003D\u001a\u0000\u03fe\u03ff"+ + "\u0001\u0000\u0000\u0000\u03ff\u0400\u0006[\u000b\u0000\u0400\u00c7\u0001"+ + "\u0000\u0000\u0000\u0401\u0402\u0003@\u0018\u0000\u0402\u0403\u0001\u0000"+ + "\u0000\u0000\u0403\u0404\u0006\\\u000b\u0000\u0404\u00c9\u0001\u0000\u0000"+ + "\u0000\u0405\u0406\u0003B\u0019\u0000\u0406\u0407\u0001\u0000\u0000\u0000"+ + "\u0407\u0408\u0006]\u000b\u0000\u0408\u00cb\u0001\u0000\u0000\u0000\u0409"+ + "\u040a\u0003F\u001b\u0000\u040a\u040b\u0001\u0000\u0000\u0000\u040b\u040c"+ + "\u0006^\u0010\u0000\u040c\u040d\u0006^\f\u0000\u040d\u00cd\u0001\u0000"+ + "\u0000\u0000\u040e\u040f\u0003\u00b2Q\u0000\u040f\u0410\u0001\u0000\u0000"+ + "\u0000\u0410\u0411\u0006_\u000e\u0000\u0411\u00cf\u0001\u0000\u0000\u0000"+ + "\u0412\u0413\u0003\u00b4R\u0000\u0413\u0414\u0001\u0000\u0000\u0000\u0414"+ + "\u0415\u0006`\u0011\u0000\u0415\u00d1\u0001\u0000\u0000\u0000\u0416\u0417"+ + "\u0003l.\u0000\u0417\u0418\u0001\u0000\u0000\u0000\u0418\u0419\u0006a"+ + "\u0012\u0000\u0419\u00d3\u0001\u0000\u0000\u0000\u041a\u041b\u0003n/\u0000"+ + "\u041b\u041c\u0001\u0000\u0000\u0000\u041c\u041d\u0006b\u0013\u0000\u041d"+ + "\u00d5\u0001\u0000\u0000\u0000\u041e\u041f\u0003h,\u0000\u041f\u0420\u0001"+ + "\u0000\u0000\u0000\u0420\u0421\u0006c\u0014\u0000\u0421\u00d7\u0001\u0000"+ + "\u0000\u0000\u0422\u0423\u0007\u0010\u0000\u0000\u0423\u0424\u0007\u0003"+ + "\u0000\u0000\u0424\u0425\u0007\u0005\u0000\u0000\u0425\u0426\u0007\f\u0000"+ + "\u0000\u0426\u0427\u0007\u0000\u0000\u0000\u0427\u0428\u0007\f\u0000\u0000"+ + "\u0428\u0429\u0007\u0005\u0000\u0000\u0429\u042a\u0007\f\u0000\u0000\u042a"+ + "\u00d9\u0001\u0000\u0000\u0000\u042b\u042f\b \u0000\u0000\u042c\u042d"+ + "\u0005/\u0000\u0000\u042d\u042f\b!\u0000\u0000\u042e\u042b\u0001\u0000"+ + "\u0000\u0000\u042e\u042c\u0001\u0000\u0000\u0000\u042f\u00db\u0001\u0000"+ + "\u0000\u0000\u0430\u0432\u0003\u00dae\u0000\u0431\u0430\u0001\u0000\u0000"+ + "\u0000\u0432\u0433\u0001\u0000\u0000\u0000\u0433\u0431\u0001\u0000\u0000"+ + "\u0000\u0433\u0434\u0001\u0000\u0000\u0000\u0434\u00dd\u0001\u0000\u0000"+ + "\u0000\u0435\u0436\u0003\u00dcf\u0000\u0436\u0437\u0001\u0000\u0000\u0000"+ + "\u0437\u0438\u0006g\u0015\u0000\u0438\u00df\u0001\u0000\u0000\u0000\u0439"+ + "\u043a\u0003\\&\u0000\u043a\u043b\u0001\u0000\u0000\u0000\u043b\u043c"+ + "\u0006h\u0016\u0000\u043c\u00e1\u0001\u0000\u0000\u0000\u043d\u043e\u0003"+ + "@\u0018\u0000\u043e\u043f\u0001\u0000\u0000\u0000\u043f\u0440\u0006i\u000b"+ + "\u0000\u0440\u00e3\u0001\u0000\u0000\u0000\u0441\u0442\u0003B\u0019\u0000"+ + "\u0442\u0443\u0001\u0000\u0000\u0000\u0443\u0444\u0006j\u000b\u0000\u0444"+ + "\u00e5\u0001\u0000\u0000\u0000\u0445\u0446\u0003D\u001a\u0000\u0446\u0447"+ + "\u0001\u0000\u0000\u0000\u0447\u0448\u0006k\u000b\u0000\u0448\u00e7\u0001"+ + "\u0000\u0000\u0000\u0449\u044a\u0003F\u001b\u0000\u044a\u044b\u0001\u0000"+ + "\u0000\u0000\u044b\u044c\u0006l\u0010\u0000\u044c\u044d\u0006l\f\u0000"+ + "\u044d\u00e9\u0001\u0000\u0000\u0000\u044e\u044f\u0003r1\u0000\u044f\u0450"+ + "\u0001\u0000\u0000\u0000\u0450\u0451\u0006m\u0017\u0000\u0451\u00eb\u0001"+ + "\u0000\u0000\u0000\u0452\u0453\u0003n/\u0000\u0453\u0454\u0001\u0000\u0000"+ + "\u0000\u0454\u0455\u0006n\u0013\u0000\u0455\u00ed\u0001\u0000\u0000\u0000"+ + "\u0456\u0457\u0003\u008a=\u0000\u0457\u0458\u0001\u0000\u0000\u0000\u0458"+ + "\u0459\u0006o\u0018\u0000\u0459\u00ef\u0001\u0000\u0000\u0000\u045a\u045b"+ + "\u0003\u00b0P\u0000\u045b\u045c\u0001\u0000\u0000\u0000\u045c\u045d\u0006"+ + "p\u0019\u0000\u045d\u00f1\u0001\u0000\u0000\u0000\u045e\u0463\u0003J\u001d"+ + "\u0000\u045f\u0463\u0003H\u001c\u0000\u0460\u0463\u0003X$\u0000\u0461"+ + "\u0463\u0003\u00a4J\u0000\u0462\u045e\u0001\u0000\u0000\u0000\u0462\u045f"+ + "\u0001\u0000\u0000\u0000\u0462\u0460\u0001\u0000\u0000\u0000\u0462\u0461"+ + "\u0001\u0000\u0000\u0000\u0463\u00f3\u0001\u0000\u0000\u0000\u0464\u0467"+ + "\u0003J\u001d\u0000\u0465\u0467\u0003\u00a4J\u0000\u0466\u0464\u0001\u0000"+ + "\u0000\u0000\u0466\u0465\u0001\u0000\u0000\u0000\u0467\u046b\u0001\u0000"+ + "\u0000\u0000\u0468\u046a\u0003\u00f2q\u0000\u0469\u0468\u0001\u0000\u0000"+ + "\u0000\u046a\u046d\u0001\u0000\u0000\u0000\u046b\u0469\u0001\u0000\u0000"+ + "\u0000\u046b\u046c\u0001\u0000\u0000\u0000\u046c\u0478\u0001\u0000\u0000"+ + "\u0000\u046d\u046b\u0001\u0000\u0000\u0000\u046e\u0471\u0003X$\u0000\u046f"+ + "\u0471\u0003R!\u0000\u0470\u046e\u0001\u0000\u0000\u0000\u0470\u046f\u0001"+ + "\u0000\u0000\u0000\u0471\u0473\u0001\u0000\u0000\u0000\u0472\u0474\u0003"+ + "\u00f2q\u0000\u0473\u0472\u0001\u0000\u0000\u0000\u0474\u0475\u0001\u0000"+ + "\u0000\u0000\u0475\u0473\u0001\u0000\u0000\u0000\u0475\u0476\u0001\u0000"+ + "\u0000\u0000\u0476\u0478\u0001\u0000\u0000\u0000\u0477\u0466\u0001\u0000"+ + "\u0000\u0000\u0477\u0470\u0001\u0000\u0000\u0000\u0478\u00f5\u0001\u0000"+ + "\u0000\u0000\u0479\u047c\u0003\u00f4r\u0000\u047a\u047c\u0003\u00b8T\u0000"+ + "\u047b\u0479\u0001\u0000\u0000\u0000\u047b\u047a\u0001\u0000\u0000\u0000"+ + "\u047c\u047d\u0001\u0000\u0000\u0000\u047d\u047b\u0001\u0000\u0000\u0000"+ + "\u047d\u047e\u0001\u0000\u0000\u0000\u047e\u00f7\u0001\u0000\u0000\u0000"+ + "\u047f\u0480\u0003@\u0018\u0000\u0480\u0481\u0001\u0000\u0000\u0000\u0481"+ + "\u0482\u0006t\u000b\u0000\u0482\u00f9\u0001\u0000\u0000\u0000\u0483\u0484"+ + "\u0003B\u0019\u0000\u0484\u0485\u0001\u0000\u0000\u0000\u0485\u0486\u0006"+ + "u\u000b\u0000\u0486\u00fb\u0001\u0000\u0000\u0000\u0487\u0488\u0003D\u001a"+ + "\u0000\u0488\u0489\u0001\u0000\u0000\u0000\u0489\u048a\u0006v\u000b\u0000"+ + "\u048a\u00fd\u0001\u0000\u0000\u0000\u048b\u048c\u0003F\u001b\u0000\u048c"+ + "\u048d\u0001\u0000\u0000\u0000\u048d\u048e\u0006w\u0010\u0000\u048e\u048f"+ + "\u0006w\f\u0000\u048f\u00ff\u0001\u0000\u0000\u0000\u0490\u0491\u0003"+ + "h,\u0000\u0491\u0492\u0001\u0000\u0000\u0000\u0492\u0493\u0006x\u0014"+ + "\u0000\u0493\u0101\u0001\u0000\u0000\u0000\u0494\u0495\u0003n/\u0000\u0495"+ + "\u0496\u0001\u0000\u0000\u0000\u0496\u0497\u0006y\u0013\u0000\u0497\u0103"+ + "\u0001\u0000\u0000\u0000\u0498\u0499\u0003r1\u0000\u0499\u049a\u0001\u0000"+ + "\u0000\u0000\u049a\u049b\u0006z\u0017\u0000\u049b\u0105\u0001\u0000\u0000"+ + "\u0000\u049c\u049d\u0003\u008a=\u0000\u049d\u049e\u0001\u0000\u0000\u0000"+ + "\u049e\u049f\u0006{\u0018\u0000\u049f\u0107\u0001\u0000\u0000\u0000\u04a0"+ + "\u04a1\u0003\u00b0P\u0000\u04a1\u04a2\u0001\u0000\u0000\u0000\u04a2\u04a3"+ + "\u0006|\u0019\u0000\u04a3\u0109\u0001\u0000\u0000\u0000\u04a4\u04a5\u0007"+ + "\f\u0000\u0000\u04a5\u04a6\u0007\u0002\u0000\u0000\u04a6\u010b\u0001\u0000"+ + "\u0000\u0000\u04a7\u04a8\u0003\u00f6s\u0000\u04a8\u04a9\u0001\u0000\u0000"+ + "\u0000\u04a9\u04aa\u0006~\u001a\u0000\u04aa\u010d\u0001\u0000\u0000\u0000"+ + "\u04ab\u04ac\u0003@\u0018\u0000\u04ac\u04ad\u0001\u0000\u0000\u0000\u04ad"+ + "\u04ae\u0006\u007f\u000b\u0000\u04ae\u010f\u0001\u0000\u0000\u0000\u04af"+ + "\u04b0\u0003B\u0019\u0000\u04b0\u04b1\u0001\u0000\u0000\u0000\u04b1\u04b2"+ + "\u0006\u0080\u000b\u0000\u04b2\u0111\u0001\u0000\u0000\u0000\u04b3\u04b4"+ + "\u0003D\u001a\u0000\u04b4\u04b5\u0001\u0000\u0000\u0000\u04b5\u04b6\u0006"+ + "\u0081\u000b\u0000\u04b6\u0113\u0001\u0000\u0000\u0000\u04b7\u04b8\u0003"+ + "F\u001b\u0000\u04b8\u04b9\u0001\u0000\u0000\u0000\u04b9\u04ba\u0006\u0082"+ + "\u0010\u0000\u04ba\u04bb\u0006\u0082\f\u0000\u04bb\u0115\u0001\u0000\u0000"+ + "\u0000\u04bc\u04bd\u0003\u00b2Q\u0000\u04bd\u04be\u0001\u0000\u0000\u0000"+ + "\u04be\u04bf\u0006\u0083\u000e\u0000\u04bf\u04c0\u0006\u0083\u001b\u0000"+ + "\u04c0\u0117\u0001\u0000\u0000\u0000\u04c1\u04c2\u0007\u0007\u0000\u0000"+ + "\u04c2\u04c3\u0007\t\u0000\u0000\u04c3\u04c4\u0001\u0000\u0000\u0000\u04c4"+ + "\u04c5\u0006\u0084\u001c\u0000\u04c5\u0119\u0001\u0000\u0000\u0000\u04c6"+ + "\u04c7\u0007\u0013\u0000\u0000\u04c7\u04c8\u0007\u0001\u0000\u0000\u04c8"+ + "\u04c9\u0007\u0005\u0000\u0000\u04c9\u04ca\u0007\n\u0000\u0000\u04ca\u04cb"+ + "\u0001\u0000\u0000\u0000\u04cb\u04cc\u0006\u0085\u001c\u0000\u04cc\u011b"+ + "\u0001\u0000\u0000\u0000\u04cd\u04ce\b\"\u0000\u0000\u04ce\u011d\u0001"+ + "\u0000\u0000\u0000\u04cf\u04d1\u0003\u011c\u0086\u0000\u04d0\u04cf\u0001"+ + "\u0000\u0000\u0000\u04d1\u04d2\u0001\u0000\u0000\u0000\u04d2\u04d0\u0001"+ + "\u0000\u0000\u0000\u04d2\u04d3\u0001\u0000\u0000\u0000\u04d3\u04d4\u0001"+ + "\u0000\u0000\u0000\u04d4\u04d5\u0003l.\u0000\u04d5\u04d7\u0001\u0000\u0000"+ + "\u0000\u04d6\u04d0\u0001\u0000\u0000\u0000\u04d6\u04d7\u0001\u0000\u0000"+ + "\u0000\u04d7\u04d9\u0001\u0000\u0000\u0000\u04d8\u04da\u0003\u011c\u0086"+ + "\u0000\u04d9\u04d8\u0001\u0000\u0000\u0000\u04da\u04db\u0001\u0000\u0000"+ + "\u0000\u04db\u04d9\u0001\u0000\u0000\u0000\u04db\u04dc\u0001\u0000\u0000"+ + "\u0000\u04dc\u011f\u0001\u0000\u0000\u0000\u04dd\u04de\u0003\u011e\u0087"+ + "\u0000\u04de\u04df\u0001\u0000\u0000\u0000\u04df\u04e0\u0006\u0088\u001d"+ + "\u0000\u04e0\u0121\u0001\u0000\u0000\u0000\u04e1\u04e2\u0003@\u0018\u0000"+ + "\u04e2\u04e3\u0001\u0000\u0000\u0000\u04e3\u04e4\u0006\u0089\u000b\u0000"+ + "\u04e4\u0123\u0001\u0000\u0000\u0000\u04e5\u04e6\u0003B\u0019\u0000\u04e6"+ + "\u04e7\u0001\u0000\u0000\u0000\u04e7\u04e8\u0006\u008a\u000b\u0000\u04e8"+ + "\u0125\u0001\u0000\u0000\u0000\u04e9\u04ea\u0003D\u001a\u0000\u04ea\u04eb"+ + "\u0001\u0000\u0000\u0000\u04eb\u04ec\u0006\u008b\u000b\u0000\u04ec\u0127"+ + "\u0001\u0000\u0000\u0000\u04ed\u04ee\u0003F\u001b\u0000\u04ee\u04ef\u0001"+ + "\u0000\u0000\u0000\u04ef\u04f0\u0006\u008c\u0010\u0000\u04f0\u04f1\u0006"+ + "\u008c\f\u0000\u04f1\u04f2\u0006\u008c\f\u0000\u04f2\u0129\u0001\u0000"+ + "\u0000\u0000\u04f3\u04f4\u0003h,\u0000\u04f4\u04f5\u0001\u0000\u0000\u0000"+ + "\u04f5\u04f6\u0006\u008d\u0014\u0000\u04f6\u012b\u0001\u0000\u0000\u0000"+ + "\u04f7\u04f8\u0003n/\u0000\u04f8\u04f9\u0001\u0000\u0000\u0000\u04f9\u04fa"+ + "\u0006\u008e\u0013\u0000\u04fa\u012d\u0001\u0000\u0000\u0000\u04fb\u04fc"+ + "\u0003r1\u0000\u04fc\u04fd\u0001\u0000\u0000\u0000\u04fd\u04fe\u0006\u008f"+ + "\u0017\u0000\u04fe\u012f\u0001\u0000\u0000\u0000\u04ff\u0500\u0003\u011a"+ + "\u0085\u0000\u0500\u0501\u0001\u0000\u0000\u0000\u0501\u0502\u0006\u0090"+ + "\u001e\u0000\u0502\u0131\u0001\u0000\u0000\u0000\u0503\u0504\u0003\u00f6"+ + "s\u0000\u0504\u0505\u0001\u0000\u0000\u0000\u0505\u0506\u0006\u0091\u001a"+ + "\u0000\u0506\u0133\u0001\u0000\u0000\u0000\u0507\u0508\u0003\u00baU\u0000"+ + "\u0508\u0509\u0001\u0000\u0000\u0000\u0509\u050a\u0006\u0092\u001f\u0000"+ + "\u050a\u0135\u0001\u0000\u0000\u0000\u050b\u050c\u0003\u008a=\u0000\u050c"+ + "\u050d\u0001\u0000\u0000\u0000\u050d\u050e\u0006\u0093\u0018\u0000\u050e"+ + "\u0137\u0001\u0000\u0000\u0000\u050f\u0510\u0003\u00b0P\u0000\u0510\u0511"+ + "\u0001\u0000\u0000\u0000\u0511\u0512\u0006\u0094\u0019\u0000\u0512\u0139"+ + "\u0001\u0000\u0000\u0000\u0513\u0514\u0003@\u0018\u0000\u0514\u0515\u0001"+ + "\u0000\u0000\u0000\u0515\u0516\u0006\u0095\u000b\u0000\u0516\u013b\u0001"+ + "\u0000\u0000\u0000\u0517\u0518\u0003B\u0019\u0000\u0518\u0519\u0001\u0000"+ + "\u0000\u0000\u0519\u051a\u0006\u0096\u000b\u0000\u051a\u013d\u0001\u0000"+ + "\u0000\u0000\u051b\u051c\u0003D\u001a\u0000\u051c\u051d\u0001\u0000\u0000"+ + "\u0000\u051d\u051e\u0006\u0097\u000b\u0000\u051e\u013f\u0001\u0000\u0000"+ + "\u0000\u051f\u0520\u0003F\u001b\u0000\u0520\u0521\u0001\u0000\u0000\u0000"+ + "\u0521\u0522\u0006\u0098\u0010\u0000\u0522\u0523\u0006\u0098\f\u0000\u0523"+ + "\u0141\u0001\u0000\u0000\u0000\u0524\u0525\u0003r1\u0000\u0525\u0526\u0001"+ + "\u0000\u0000\u0000\u0526\u0527\u0006\u0099\u0017\u0000\u0527\u0143\u0001"+ + "\u0000\u0000\u0000\u0528\u0529\u0003\u008a=\u0000\u0529\u052a\u0001\u0000"+ + "\u0000\u0000\u052a\u052b\u0006\u009a\u0018\u0000\u052b\u0145\u0001\u0000"+ + "\u0000\u0000\u052c\u052d\u0003\u00b0P\u0000\u052d\u052e\u0001\u0000\u0000"+ + "\u0000\u052e\u052f\u0006\u009b\u0019\u0000\u052f\u0147\u0001\u0000\u0000"+ + "\u0000\u0530\u0531\u0003\u00baU\u0000\u0531\u0532\u0001\u0000\u0000\u0000"+ + "\u0532\u0533\u0006\u009c\u001f\u0000\u0533\u0149\u0001\u0000\u0000\u0000"+ + "\u0534\u0535\u0003\u00b6S\u0000\u0535\u0536\u0001\u0000\u0000\u0000\u0536"+ + "\u0537\u0006\u009d \u0000\u0537\u014b\u0001\u0000\u0000\u0000\u0538\u0539"+ + "\u0003@\u0018\u0000\u0539\u053a\u0001\u0000\u0000\u0000\u053a\u053b\u0006"+ + "\u009e\u000b\u0000\u053b\u014d\u0001\u0000\u0000\u0000\u053c\u053d\u0003"+ + "B\u0019\u0000\u053d\u053e\u0001\u0000\u0000\u0000\u053e\u053f\u0006\u009f"+ + "\u000b\u0000\u053f\u014f\u0001\u0000\u0000\u0000\u0540\u0541\u0003D\u001a"+ + "\u0000\u0541\u0542\u0001\u0000\u0000\u0000\u0542\u0543\u0006\u00a0\u000b"+ + "\u0000\u0543\u0151\u0001\u0000\u0000\u0000\u0544\u0545\u0003F\u001b\u0000"+ + "\u0545\u0546\u0001\u0000\u0000\u0000\u0546\u0547\u0006\u00a1\u0010\u0000"+ + "\u0547\u0548\u0006\u00a1\f\u0000\u0548\u0153\u0001\u0000\u0000\u0000\u0549"+ + "\u054a\u0007\u0001\u0000\u0000\u054a\u054b\u0007\t\u0000\u0000\u054b\u054c"+ + "\u0007\u000f\u0000\u0000\u054c\u054d\u0007\u0007\u0000\u0000\u054d\u0155"+ + "\u0001\u0000\u0000\u0000\u054e\u054f\u0003@\u0018\u0000\u054f\u0550\u0001"+ + "\u0000\u0000\u0000\u0550\u0551\u0006\u00a3\u000b\u0000\u0551\u0157\u0001"+ + "\u0000\u0000\u0000\u0552\u0553\u0003B\u0019\u0000\u0553\u0554\u0001\u0000"+ + "\u0000\u0000\u0554\u0555\u0006\u00a4\u000b\u0000\u0555\u0159\u0001\u0000"+ + "\u0000\u0000\u0556\u0557\u0003D\u001a\u0000\u0557\u0558\u0001\u0000\u0000"+ + "\u0000\u0558\u0559\u0006\u00a5\u000b\u0000\u0559\u015b\u0001\u0000\u0000"+ + "\u0000\u055a\u055b\u0003\u00b4R\u0000\u055b\u055c\u0001\u0000\u0000\u0000"+ + "\u055c\u055d\u0006\u00a6\u0011\u0000\u055d\u055e\u0006\u00a6\f\u0000\u055e"+ + "\u015d\u0001\u0000\u0000\u0000\u055f\u0560\u0003l.\u0000\u0560\u0561\u0001"+ + "\u0000\u0000\u0000\u0561\u0562\u0006\u00a7\u0012\u0000\u0562\u015f\u0001"+ + "\u0000\u0000\u0000\u0563\u0569\u0003R!\u0000\u0564\u0569\u0003H\u001c"+ + "\u0000\u0565\u0569\u0003r1\u0000\u0566\u0569\u0003J\u001d\u0000\u0567"+ + "\u0569\u0003X$\u0000\u0568\u0563\u0001\u0000\u0000\u0000\u0568\u0564\u0001"+ + "\u0000\u0000\u0000\u0568\u0565\u0001\u0000\u0000\u0000\u0568\u0566\u0001"+ + "\u0000\u0000\u0000\u0568\u0567\u0001\u0000\u0000\u0000\u0569\u056a\u0001"+ + "\u0000\u0000\u0000\u056a\u0568\u0001\u0000\u0000\u0000\u056a\u056b\u0001"+ + "\u0000\u0000\u0000\u056b\u0161\u0001\u0000\u0000\u0000\u056c\u056d\u0003"+ + "@\u0018\u0000\u056d\u056e\u0001\u0000\u0000\u0000\u056e\u056f\u0006\u00a9"+ + "\u000b\u0000\u056f\u0163\u0001\u0000\u0000\u0000\u0570\u0571\u0003B\u0019"+ + "\u0000\u0571\u0572\u0001\u0000\u0000\u0000\u0572\u0573\u0006\u00aa\u000b"+ + "\u0000\u0573\u0165\u0001\u0000\u0000\u0000\u0574\u0575\u0003D\u001a\u0000"+ + "\u0575\u0576\u0001\u0000\u0000\u0000\u0576\u0577\u0006\u00ab\u000b\u0000"+ + "\u0577\u0167\u0001\u0000\u0000\u0000\u0578\u0579\u0003F\u001b\u0000\u0579"+ + "\u057a\u0001\u0000\u0000\u0000\u057a\u057b\u0006\u00ac\u0010\u0000\u057b"+ + "\u057c\u0006\u00ac\f\u0000\u057c\u0169\u0001\u0000\u0000\u0000\u057d\u057e"+ + "\u0003l.\u0000\u057e\u057f\u0001\u0000\u0000\u0000\u057f\u0580\u0006\u00ad"+ + "\u0012\u0000\u0580\u016b\u0001\u0000\u0000\u0000\u0581\u0582\u0003n/\u0000"+ + "\u0582\u0583\u0001\u0000\u0000\u0000\u0583\u0584\u0006\u00ae\u0013\u0000"+ + "\u0584\u016d\u0001\u0000\u0000\u0000\u0585\u0586\u0003r1\u0000\u0586\u0587"+ + "\u0001\u0000\u0000\u0000\u0587\u0588\u0006\u00af\u0017\u0000\u0588\u016f"+ + "\u0001\u0000\u0000\u0000\u0589\u058a\u0003\u0118\u0084\u0000\u058a\u058b"+ + "\u0001\u0000\u0000\u0000\u058b\u058c\u0006\u00b0!\u0000\u058c\u058d\u0006"+ + "\u00b0\"\u0000\u058d\u0171\u0001\u0000\u0000\u0000\u058e\u058f\u0003\u00dc"+ + "f\u0000\u058f\u0590\u0001\u0000\u0000\u0000\u0590\u0591\u0006\u00b1\u0015"+ + "\u0000\u0591\u0173\u0001\u0000\u0000\u0000\u0592\u0593\u0003\\&\u0000"+ + "\u0593\u0594\u0001\u0000\u0000\u0000\u0594\u0595\u0006\u00b2\u0016\u0000"+ + "\u0595\u0175\u0001\u0000\u0000\u0000\u0596\u0597\u0003@\u0018\u0000\u0597"+ + "\u0598\u0001\u0000\u0000\u0000\u0598\u0599\u0006\u00b3\u000b\u0000\u0599"+ + "\u0177\u0001\u0000\u0000\u0000\u059a\u059b\u0003B\u0019\u0000\u059b\u059c"+ + "\u0001\u0000\u0000\u0000\u059c\u059d\u0006\u00b4\u000b\u0000\u059d\u0179"+ + "\u0001\u0000\u0000\u0000\u059e\u059f\u0003D\u001a\u0000\u059f\u05a0\u0001"+ + "\u0000\u0000\u0000\u05a0\u05a1\u0006\u00b5\u000b\u0000\u05a1\u017b\u0001"+ + "\u0000\u0000\u0000\u05a2\u05a3\u0003F\u001b\u0000\u05a3\u05a4\u0001\u0000"+ + "\u0000\u0000\u05a4\u05a5\u0006\u00b6\u0010\u0000\u05a5\u05a6\u0006\u00b6"+ + "\f\u0000\u05a6\u05a7\u0006\u00b6\f\u0000\u05a7\u017d\u0001\u0000\u0000"+ + "\u0000\u05a8\u05a9\u0003n/\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa"+ + "\u05ab\u0006\u00b7\u0013\u0000\u05ab\u017f\u0001\u0000\u0000\u0000\u05ac"+ + "\u05ad\u0003r1\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006"+ + "\u00b8\u0017\u0000\u05af\u0181\u0001\u0000\u0000\u0000\u05b0\u05b1\u0003"+ + "\u00f6s\u0000\u05b1\u05b2\u0001\u0000\u0000\u0000\u05b2\u05b3\u0006\u00b9"+ + "\u001a\u0000\u05b3\u0183\u0001\u0000\u0000\u0000\u05b4\u05b5\u0003@\u0018"+ + "\u0000\u05b5\u05b6\u0001\u0000\u0000\u0000\u05b6\u05b7\u0006\u00ba\u000b"+ + "\u0000\u05b7\u0185\u0001\u0000\u0000\u0000\u05b8\u05b9\u0003B\u0019\u0000"+ + "\u05b9\u05ba\u0001\u0000\u0000\u0000\u05ba\u05bb\u0006\u00bb\u000b\u0000"+ + "\u05bb\u0187\u0001\u0000\u0000\u0000\u05bc\u05bd\u0003D\u001a\u0000\u05bd"+ + "\u05be\u0001\u0000\u0000\u0000\u05be\u05bf\u0006\u00bc\u000b\u0000\u05bf"+ + "\u0189\u0001\u0000\u0000\u0000\u05c0\u05c1\u0003F\u001b\u0000\u05c1\u05c2"+ + "\u0001\u0000\u0000\u0000\u05c2\u05c3\u0006\u00bd\u0010\u0000\u05c3\u05c4"+ + "\u0006\u00bd\f\u0000\u05c4\u018b\u0001\u0000\u0000\u0000\u05c5\u05c6\u0007"+ + "#\u0000\u0000\u05c6\u05c7\u0007\u0007\u0000\u0000\u05c7\u05c8\u0007\u0001"+ + "\u0000\u0000\u05c8\u05c9\u0007\t\u0000\u0000\u05c9\u018d\u0001\u0000\u0000"+ + "\u0000\u05ca\u05cb\u0003\u010a}\u0000\u05cb\u05cc\u0001\u0000\u0000\u0000"+ + "\u05cc\u05cd\u0006\u00bf#\u0000\u05cd\u018f\u0001\u0000\u0000\u0000\u05ce"+ + "\u05cf\u0003\u0118\u0084\u0000\u05cf\u05d0\u0001\u0000\u0000\u0000\u05d0"+ + "\u05d1\u0006\u00c0!\u0000\u05d1\u05d2\u0006\u00c0\f\u0000\u05d2\u05d3"+ + "\u0006\u00c0\u0000\u0000\u05d3\u0191\u0001\u0000\u0000\u0000\u05d4\u05d5"+ + "\u0007\u0014\u0000\u0000\u05d5\u05d6\u0007\u0002\u0000\u0000\u05d6\u05d7"+ + "\u0007\u0001\u0000\u0000\u05d7\u05d8\u0007\t\u0000\u0000\u05d8\u05d9\u0007"+ + "\u0011\u0000\u0000\u05d9\u05da\u0001\u0000\u0000\u0000\u05da\u05db\u0006"+ + "\u00c1\f\u0000\u05db\u05dc\u0006\u00c1\u0000\u0000\u05dc\u0193\u0001\u0000"+ + "\u0000\u0000\u05dd\u05de\u0003\u00dcf\u0000\u05de\u05df\u0001\u0000\u0000"+ + "\u0000\u05df\u05e0\u0006\u00c2\u0015\u0000\u05e0\u0195\u0001\u0000\u0000"+ + "\u0000\u05e1\u05e2\u0003\\&\u0000\u05e2\u05e3\u0001\u0000\u0000\u0000"+ + "\u05e3\u05e4\u0006\u00c3\u0016\u0000\u05e4\u0197\u0001\u0000\u0000\u0000"+ + "\u05e5\u05e6\u0003l.\u0000\u05e6\u05e7\u0001\u0000\u0000\u0000\u05e7\u05e8"+ + "\u0006\u00c4\u0012\u0000\u05e8\u0199\u0001\u0000\u0000\u0000\u05e9\u05ea"+ + "\u0003\u00b6S\u0000\u05ea\u05eb\u0001\u0000\u0000\u0000\u05eb\u05ec\u0006"+ + "\u00c5 \u0000\u05ec\u019b\u0001\u0000\u0000\u0000\u05ed\u05ee\u0003\u00ba"+ + "U\u0000\u05ee\u05ef\u0001\u0000\u0000\u0000\u05ef\u05f0\u0006\u00c6\u001f"+ + "\u0000\u05f0\u019d\u0001\u0000\u0000\u0000\u05f1\u05f2\u0003@\u0018\u0000"+ + "\u05f2\u05f3\u0001\u0000\u0000\u0000\u05f3\u05f4\u0006\u00c7\u000b\u0000"+ + "\u05f4\u019f\u0001\u0000\u0000\u0000\u05f5\u05f6\u0003B\u0019\u0000\u05f6"+ + "\u05f7\u0001\u0000\u0000\u0000\u05f7\u05f8\u0006\u00c8\u000b\u0000\u05f8"+ + "\u01a1\u0001\u0000\u0000\u0000\u05f9\u05fa\u0003D\u001a\u0000\u05fa\u05fb"+ + "\u0001\u0000\u0000\u0000\u05fb\u05fc\u0006\u00c9\u000b\u0000\u05fc\u01a3"+ + "\u0001\u0000\u0000\u0000\u05fd\u05fe\u0003F\u001b\u0000\u05fe\u05ff\u0001"+ + "\u0000\u0000\u0000\u05ff\u0600\u0006\u00ca\u0010\u0000\u0600\u0601\u0006"+ + "\u00ca\f\u0000\u0601\u01a5\u0001\u0000\u0000\u0000\u0602\u0603\u0003\u00dc"+ + "f\u0000\u0603\u0604\u0001\u0000\u0000\u0000\u0604\u0605\u0006\u00cb\u0015"+ + "\u0000\u0605\u0606\u0006\u00cb\f\u0000\u0606\u0607\u0006\u00cb$\u0000"+ + "\u0607\u01a7\u0001\u0000\u0000\u0000\u0608\u0609\u0003\\&\u0000\u0609"+ + "\u060a\u0001\u0000\u0000\u0000\u060a\u060b\u0006\u00cc\u0016\u0000\u060b"+ + "\u060c\u0006\u00cc\f\u0000\u060c\u060d\u0006\u00cc$\u0000\u060d\u01a9"+ + "\u0001\u0000\u0000\u0000\u060e\u060f\u0003@\u0018\u0000\u060f\u0610\u0001"+ + "\u0000\u0000\u0000\u0610\u0611\u0006\u00cd\u000b\u0000\u0611\u01ab\u0001"+ + "\u0000\u0000\u0000\u0612\u0613\u0003B\u0019\u0000\u0613\u0614\u0001\u0000"+ + "\u0000\u0000\u0614\u0615\u0006\u00ce\u000b\u0000\u0615\u01ad\u0001\u0000"+ + "\u0000\u0000\u0616\u0617\u0003D\u001a\u0000\u0617\u0618\u0001\u0000\u0000"+ + "\u0000\u0618\u0619\u0006\u00cf\u000b\u0000\u0619\u01af\u0001\u0000\u0000"+ + "\u0000\u061a\u061b\u0003l.\u0000\u061b\u061c\u0001\u0000\u0000\u0000\u061c"+ + "\u061d\u0006\u00d0\u0012\u0000\u061d\u061e\u0006\u00d0\f\u0000\u061e\u061f"+ + "\u0006\u00d0\n\u0000\u061f\u01b1\u0001\u0000\u0000\u0000\u0620\u0621\u0003"+ + "n/\u0000\u0621\u0622\u0001\u0000\u0000\u0000\u0622\u0623\u0006\u00d1\u0013"+ + "\u0000\u0623\u0624\u0006\u00d1\f\u0000\u0624\u0625\u0006\u00d1\n\u0000"+ + "\u0625\u01b3\u0001\u0000\u0000\u0000\u0626\u0627\u0003@\u0018\u0000\u0627"+ + "\u0628\u0001\u0000\u0000\u0000\u0628\u0629\u0006\u00d2\u000b\u0000\u0629"+ + "\u01b5\u0001\u0000\u0000\u0000\u062a\u062b\u0003B\u0019\u0000\u062b\u062c"+ + "\u0001\u0000\u0000\u0000\u062c\u062d\u0006\u00d3\u000b\u0000\u062d\u01b7"+ + "\u0001\u0000\u0000\u0000\u062e\u062f\u0003D\u001a\u0000\u062f\u0630\u0001"+ + "\u0000\u0000\u0000\u0630\u0631\u0006\u00d4\u000b\u0000\u0631\u01b9\u0001"+ + "\u0000\u0000\u0000\u0632\u0633\u0003\u00baU\u0000\u0633\u0634\u0001\u0000"+ + "\u0000\u0000\u0634\u0635\u0006\u00d5\f\u0000\u0635\u0636\u0006\u00d5\u0000"+ + "\u0000\u0636\u0637\u0006\u00d5\u001f\u0000\u0637\u01bb\u0001\u0000\u0000"+ + "\u0000\u0638\u0639\u0003\u00b6S\u0000\u0639\u063a\u0001\u0000\u0000\u0000"+ + "\u063a\u063b\u0006\u00d6\f\u0000\u063b\u063c\u0006\u00d6\u0000\u0000\u063c"+ + "\u063d\u0006\u00d6 \u0000\u063d\u01bd\u0001\u0000\u0000\u0000\u063e\u063f"+ + "\u0003b)\u0000\u063f\u0640\u0001\u0000\u0000\u0000\u0640\u0641\u0006\u00d7"+ + "\f\u0000\u0641\u0642\u0006\u00d7\u0000\u0000\u0642\u0643\u0006\u00d7%"+ + "\u0000\u0643\u01bf\u0001\u0000\u0000\u0000\u0644\u0645\u0003F\u001b\u0000"+ + "\u0645\u0646\u0001\u0000\u0000\u0000\u0646\u0647\u0006\u00d8\u0010\u0000"+ + "\u0647\u0648\u0006\u00d8\f\u0000\u0648\u01c1\u0001\u0000\u0000\u0000B"+ "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ - "\u000f\u0299\u02a3\u02a7\u02aa\u02b3\u02b5\u02c0\u02d3\u02d8\u02e1\u02e8"+ - "\u02ed\u02ef\u02fa\u0302\u0305\u0307\u030c\u0311\u0317\u031e\u0323\u0329"+ - "\u032c\u0334\u0338\u03bc\u03c1\u03c8\u03ca\u03da\u03df\u03e4\u03e6\u03ec"+ - "\u0439\u043e\u046f\u0473\u0478\u047d\u0482\u0484\u0488\u048a\u04e1\u04e5"+ - "\u04ea\u057b\u057d\'\u0005\u0001\u0000\u0005\u0004\u0000\u0005\u0006\u0000"+ + "\u000f\u028e\u0298\u029c\u029f\u02a8\u02aa\u02b5\u02c8\u02cd\u02d6\u02dd"+ + "\u02e2\u02e4\u02ef\u02f7\u02fa\u02fc\u0301\u0306\u030c\u0313\u0318\u031e"+ + "\u0321\u0329\u032d\u03b1\u03b6\u03bd\u03bf\u03cf\u03d4\u03d9\u03db\u03e1"+ + "\u042e\u0433\u0462\u0466\u046b\u0470\u0475\u0477\u047b\u047d\u04d2\u04d6"+ + "\u04db\u0568\u056a&\u0005\u0001\u0000\u0005\u0004\u0000\u0005\u0006\u0000"+ "\u0005\u0002\u0000\u0005\u0003\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005"+ - "\t\u0000\u0005\u000b\u0000\u0005\u000e\u0000\u0005\r\u0000\u0000\u0001"+ - "\u0000\u0004\u0000\u0000\u0007\u0010\u0000\u0007H\u0000\u0005\u0000\u0000"+ - "\u0007\u001d\u0000\u0007I\u0000\u0007&\u0000\u0007\'\u0000\u0007$\u0000"+ - "\u0007S\u0000\u0007\u001e\u0000\u0007)\u0000\u00075\u0000\u0007G\u0000"+ - "\u0007W\u0000\u0005\n\u0000\u0005\u0007\u0000\u0007a\u0000\u0007`\u0000"+ - "\u0007K\u0000\u0007J\u0000\u0007_\u0000\u0005\f\u0000\u0007\u0014\u0000"+ - "\u0007[\u0000\u0005\u000f\u0000\u0007!\u0000"; + "\t\u0000\u0005\r\u0000\u0005\u000b\u0000\u0005\u000e\u0000\u0000\u0001"+ + "\u0000\u0004\u0000\u0000\u0007\u0010\u0000\u0007G\u0000\u0005\u0000\u0000"+ + "\u0007\u001c\u0000\u0007H\u0000\u0007%\u0000\u0007&\u0000\u0007#\u0000"+ + "\u0007R\u0000\u0007\u001d\u0000\u0007(\u0000\u00074\u0000\u0007F\u0000"+ + "\u0007V\u0000\u0005\n\u0000\u0005\u0007\u0000\u0007`\u0000\u0007_\u0000"+ + "\u0007J\u0000\u0007I\u0000\u0007^\u0000\u0005\f\u0000\u0007Z\u0000\u0005"+ + "\u000f\u0000\u0007 \u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index ef93e3961d842..bdd69c1794395 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -16,8 +16,7 @@ null 'sort' 'stats' 'where' -null -null +'lookup' null null null @@ -120,6 +119,7 @@ null null null null +'join' 'USING' null null @@ -149,14 +149,13 @@ SHOW SORT STATS WHERE +JOIN_LOOKUP DEV_INLINESTATS DEV_LOOKUP DEV_METRICS -DEV_JOIN DEV_JOIN_FULL DEV_JOIN_LEFT DEV_JOIN_RIGHT -DEV_JOIN_LOOKUP UNKNOWN_CMD LINE_COMMENT MULTILINE_COMMENT @@ -253,6 +252,7 @@ LOOKUP_WS LOOKUP_FIELD_LINE_COMMENT LOOKUP_FIELD_MULTILINE_COMMENT LOOKUP_FIELD_WS +JOIN USING JOIN_LINE_COMMENT JOIN_MULTILINE_COMMENT @@ -336,4 +336,4 @@ joinPredicate atn: -[4, 1, 130, 662, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 146, 8, 1, 10, 1, 12, 1, 149, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 157, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 177, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 189, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 196, 8, 5, 10, 5, 12, 5, 199, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 206, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 211, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 219, 8, 5, 10, 5, 12, 5, 222, 9, 5, 1, 6, 1, 6, 3, 6, 226, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 233, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 238, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 243, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 253, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 259, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 267, 8, 9, 10, 9, 12, 9, 270, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 280, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 285, 8, 10, 10, 10, 12, 10, 288, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 296, 8, 11, 10, 11, 12, 11, 299, 9, 11, 1, 11, 1, 11, 3, 11, 303, 8, 11, 3, 11, 305, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 5, 13, 315, 8, 13, 10, 13, 12, 13, 318, 9, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 5, 17, 334, 8, 17, 10, 17, 12, 17, 337, 9, 17, 1, 18, 1, 18, 1, 18, 3, 18, 342, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 350, 8, 19, 10, 19, 12, 19, 353, 9, 19, 1, 19, 3, 19, 356, 8, 19, 1, 20, 1, 20, 1, 20, 3, 20, 361, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 3, 23, 371, 8, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 377, 8, 24, 10, 24, 12, 24, 380, 9, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 390, 8, 26, 10, 26, 12, 26, 393, 9, 26, 1, 26, 3, 26, 396, 8, 26, 1, 26, 1, 26, 3, 26, 400, 8, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 407, 8, 28, 1, 28, 1, 28, 3, 28, 411, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 416, 8, 29, 10, 29, 12, 29, 419, 9, 29, 1, 30, 1, 30, 1, 30, 3, 30, 424, 8, 30, 1, 31, 1, 31, 1, 31, 5, 31, 429, 8, 31, 10, 31, 12, 31, 432, 9, 31, 1, 32, 1, 32, 1, 32, 5, 32, 437, 8, 32, 10, 32, 12, 32, 440, 9, 32, 1, 33, 1, 33, 1, 33, 5, 33, 445, 8, 33, 10, 33, 12, 33, 448, 9, 33, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 3, 35, 455, 8, 35, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 470, 8, 36, 10, 36, 12, 36, 473, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 481, 8, 36, 10, 36, 12, 36, 484, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 492, 8, 36, 10, 36, 12, 36, 495, 9, 36, 1, 36, 1, 36, 3, 36, 499, 8, 36, 1, 37, 1, 37, 3, 37, 503, 8, 37, 1, 38, 1, 38, 1, 38, 3, 38, 508, 8, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 517, 8, 40, 10, 40, 12, 40, 520, 9, 40, 1, 41, 1, 41, 3, 41, 524, 8, 41, 1, 41, 1, 41, 3, 41, 528, 8, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 5, 44, 540, 8, 44, 10, 44, 12, 44, 543, 9, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 3, 46, 553, 8, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 5, 49, 565, 8, 49, 10, 49, 12, 49, 568, 9, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 3, 52, 578, 8, 52, 1, 53, 3, 53, 581, 8, 53, 1, 53, 1, 53, 1, 54, 3, 54, 586, 8, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 3, 60, 608, 8, 60, 1, 60, 1, 60, 1, 60, 1, 60, 5, 60, 614, 8, 60, 10, 60, 12, 60, 617, 9, 60, 3, 60, 619, 8, 60, 1, 61, 1, 61, 1, 61, 3, 61, 624, 8, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 63, 3, 63, 637, 8, 63, 1, 64, 3, 64, 640, 8, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 3, 65, 649, 8, 65, 1, 66, 1, 66, 1, 66, 1, 66, 5, 66, 655, 8, 66, 10, 66, 12, 66, 658, 9, 66, 1, 67, 1, 67, 1, 67, 0, 4, 2, 10, 18, 20, 68, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 83, 83, 1, 0, 74, 75, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 688, 0, 136, 1, 0, 0, 0, 2, 139, 1, 0, 0, 0, 4, 156, 1, 0, 0, 0, 6, 176, 1, 0, 0, 0, 8, 178, 1, 0, 0, 0, 10, 210, 1, 0, 0, 0, 12, 237, 1, 0, 0, 0, 14, 239, 1, 0, 0, 0, 16, 252, 1, 0, 0, 0, 18, 258, 1, 0, 0, 0, 20, 279, 1, 0, 0, 0, 22, 289, 1, 0, 0, 0, 24, 308, 1, 0, 0, 0, 26, 310, 1, 0, 0, 0, 28, 321, 1, 0, 0, 0, 30, 325, 1, 0, 0, 0, 32, 327, 1, 0, 0, 0, 34, 330, 1, 0, 0, 0, 36, 341, 1, 0, 0, 0, 38, 345, 1, 0, 0, 0, 40, 360, 1, 0, 0, 0, 42, 364, 1, 0, 0, 0, 44, 366, 1, 0, 0, 0, 46, 370, 1, 0, 0, 0, 48, 372, 1, 0, 0, 0, 50, 381, 1, 0, 0, 0, 52, 385, 1, 0, 0, 0, 54, 401, 1, 0, 0, 0, 56, 404, 1, 0, 0, 0, 58, 412, 1, 0, 0, 0, 60, 420, 1, 0, 0, 0, 62, 425, 1, 0, 0, 0, 64, 433, 1, 0, 0, 0, 66, 441, 1, 0, 0, 0, 68, 449, 1, 0, 0, 0, 70, 454, 1, 0, 0, 0, 72, 498, 1, 0, 0, 0, 74, 502, 1, 0, 0, 0, 76, 507, 1, 0, 0, 0, 78, 509, 1, 0, 0, 0, 80, 512, 1, 0, 0, 0, 82, 521, 1, 0, 0, 0, 84, 529, 1, 0, 0, 0, 86, 532, 1, 0, 0, 0, 88, 535, 1, 0, 0, 0, 90, 544, 1, 0, 0, 0, 92, 548, 1, 0, 0, 0, 94, 554, 1, 0, 0, 0, 96, 558, 1, 0, 0, 0, 98, 561, 1, 0, 0, 0, 100, 569, 1, 0, 0, 0, 102, 573, 1, 0, 0, 0, 104, 577, 1, 0, 0, 0, 106, 580, 1, 0, 0, 0, 108, 585, 1, 0, 0, 0, 110, 589, 1, 0, 0, 0, 112, 591, 1, 0, 0, 0, 114, 593, 1, 0, 0, 0, 116, 596, 1, 0, 0, 0, 118, 600, 1, 0, 0, 0, 120, 603, 1, 0, 0, 0, 122, 623, 1, 0, 0, 0, 124, 627, 1, 0, 0, 0, 126, 632, 1, 0, 0, 0, 128, 639, 1, 0, 0, 0, 130, 645, 1, 0, 0, 0, 132, 650, 1, 0, 0, 0, 134, 659, 1, 0, 0, 0, 136, 137, 3, 2, 1, 0, 137, 138, 5, 0, 0, 1, 138, 1, 1, 0, 0, 0, 139, 140, 6, 1, -1, 0, 140, 141, 3, 4, 2, 0, 141, 147, 1, 0, 0, 0, 142, 143, 10, 1, 0, 0, 143, 144, 5, 29, 0, 0, 144, 146, 3, 6, 3, 0, 145, 142, 1, 0, 0, 0, 146, 149, 1, 0, 0, 0, 147, 145, 1, 0, 0, 0, 147, 148, 1, 0, 0, 0, 148, 3, 1, 0, 0, 0, 149, 147, 1, 0, 0, 0, 150, 157, 3, 114, 57, 0, 151, 157, 3, 38, 19, 0, 152, 157, 3, 32, 16, 0, 153, 157, 3, 118, 59, 0, 154, 155, 4, 2, 1, 0, 155, 157, 3, 52, 26, 0, 156, 150, 1, 0, 0, 0, 156, 151, 1, 0, 0, 0, 156, 152, 1, 0, 0, 0, 156, 153, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 5, 1, 0, 0, 0, 158, 177, 3, 54, 27, 0, 159, 177, 3, 8, 4, 0, 160, 177, 3, 84, 42, 0, 161, 177, 3, 78, 39, 0, 162, 177, 3, 56, 28, 0, 163, 177, 3, 80, 40, 0, 164, 177, 3, 86, 43, 0, 165, 177, 3, 88, 44, 0, 166, 177, 3, 92, 46, 0, 167, 177, 3, 94, 47, 0, 168, 177, 3, 120, 60, 0, 169, 177, 3, 96, 48, 0, 170, 171, 4, 3, 2, 0, 171, 177, 3, 126, 63, 0, 172, 173, 4, 3, 3, 0, 173, 177, 3, 124, 62, 0, 174, 175, 4, 3, 4, 0, 175, 177, 3, 128, 64, 0, 176, 158, 1, 0, 0, 0, 176, 159, 1, 0, 0, 0, 176, 160, 1, 0, 0, 0, 176, 161, 1, 0, 0, 0, 176, 162, 1, 0, 0, 0, 176, 163, 1, 0, 0, 0, 176, 164, 1, 0, 0, 0, 176, 165, 1, 0, 0, 0, 176, 166, 1, 0, 0, 0, 176, 167, 1, 0, 0, 0, 176, 168, 1, 0, 0, 0, 176, 169, 1, 0, 0, 0, 176, 170, 1, 0, 0, 0, 176, 172, 1, 0, 0, 0, 176, 174, 1, 0, 0, 0, 177, 7, 1, 0, 0, 0, 178, 179, 5, 16, 0, 0, 179, 180, 3, 10, 5, 0, 180, 9, 1, 0, 0, 0, 181, 182, 6, 5, -1, 0, 182, 183, 5, 49, 0, 0, 183, 211, 3, 10, 5, 8, 184, 211, 3, 16, 8, 0, 185, 211, 3, 12, 6, 0, 186, 188, 3, 16, 8, 0, 187, 189, 5, 49, 0, 0, 188, 187, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 191, 5, 44, 0, 0, 191, 192, 5, 48, 0, 0, 192, 197, 3, 16, 8, 0, 193, 194, 5, 39, 0, 0, 194, 196, 3, 16, 8, 0, 195, 193, 1, 0, 0, 0, 196, 199, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 200, 1, 0, 0, 0, 199, 197, 1, 0, 0, 0, 200, 201, 5, 55, 0, 0, 201, 211, 1, 0, 0, 0, 202, 203, 3, 16, 8, 0, 203, 205, 5, 45, 0, 0, 204, 206, 5, 49, 0, 0, 205, 204, 1, 0, 0, 0, 205, 206, 1, 0, 0, 0, 206, 207, 1, 0, 0, 0, 207, 208, 5, 50, 0, 0, 208, 211, 1, 0, 0, 0, 209, 211, 3, 14, 7, 0, 210, 181, 1, 0, 0, 0, 210, 184, 1, 0, 0, 0, 210, 185, 1, 0, 0, 0, 210, 186, 1, 0, 0, 0, 210, 202, 1, 0, 0, 0, 210, 209, 1, 0, 0, 0, 211, 220, 1, 0, 0, 0, 212, 213, 10, 5, 0, 0, 213, 214, 5, 34, 0, 0, 214, 219, 3, 10, 5, 6, 215, 216, 10, 4, 0, 0, 216, 217, 5, 52, 0, 0, 217, 219, 3, 10, 5, 5, 218, 212, 1, 0, 0, 0, 218, 215, 1, 0, 0, 0, 219, 222, 1, 0, 0, 0, 220, 218, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 11, 1, 0, 0, 0, 222, 220, 1, 0, 0, 0, 223, 225, 3, 16, 8, 0, 224, 226, 5, 49, 0, 0, 225, 224, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 1, 0, 0, 0, 227, 228, 5, 47, 0, 0, 228, 229, 3, 110, 55, 0, 229, 238, 1, 0, 0, 0, 230, 232, 3, 16, 8, 0, 231, 233, 5, 49, 0, 0, 232, 231, 1, 0, 0, 0, 232, 233, 1, 0, 0, 0, 233, 234, 1, 0, 0, 0, 234, 235, 5, 54, 0, 0, 235, 236, 3, 110, 55, 0, 236, 238, 1, 0, 0, 0, 237, 223, 1, 0, 0, 0, 237, 230, 1, 0, 0, 0, 238, 13, 1, 0, 0, 0, 239, 242, 3, 62, 31, 0, 240, 241, 5, 37, 0, 0, 241, 243, 3, 30, 15, 0, 242, 240, 1, 0, 0, 0, 242, 243, 1, 0, 0, 0, 243, 244, 1, 0, 0, 0, 244, 245, 5, 38, 0, 0, 245, 246, 3, 72, 36, 0, 246, 15, 1, 0, 0, 0, 247, 253, 3, 18, 9, 0, 248, 249, 3, 18, 9, 0, 249, 250, 3, 112, 56, 0, 250, 251, 3, 18, 9, 0, 251, 253, 1, 0, 0, 0, 252, 247, 1, 0, 0, 0, 252, 248, 1, 0, 0, 0, 253, 17, 1, 0, 0, 0, 254, 255, 6, 9, -1, 0, 255, 259, 3, 20, 10, 0, 256, 257, 7, 0, 0, 0, 257, 259, 3, 18, 9, 3, 258, 254, 1, 0, 0, 0, 258, 256, 1, 0, 0, 0, 259, 268, 1, 0, 0, 0, 260, 261, 10, 2, 0, 0, 261, 262, 7, 1, 0, 0, 262, 267, 3, 18, 9, 3, 263, 264, 10, 1, 0, 0, 264, 265, 7, 0, 0, 0, 265, 267, 3, 18, 9, 2, 266, 260, 1, 0, 0, 0, 266, 263, 1, 0, 0, 0, 267, 270, 1, 0, 0, 0, 268, 266, 1, 0, 0, 0, 268, 269, 1, 0, 0, 0, 269, 19, 1, 0, 0, 0, 270, 268, 1, 0, 0, 0, 271, 272, 6, 10, -1, 0, 272, 280, 3, 72, 36, 0, 273, 280, 3, 62, 31, 0, 274, 280, 3, 22, 11, 0, 275, 276, 5, 48, 0, 0, 276, 277, 3, 10, 5, 0, 277, 278, 5, 55, 0, 0, 278, 280, 1, 0, 0, 0, 279, 271, 1, 0, 0, 0, 279, 273, 1, 0, 0, 0, 279, 274, 1, 0, 0, 0, 279, 275, 1, 0, 0, 0, 280, 286, 1, 0, 0, 0, 281, 282, 10, 1, 0, 0, 282, 283, 5, 37, 0, 0, 283, 285, 3, 30, 15, 0, 284, 281, 1, 0, 0, 0, 285, 288, 1, 0, 0, 0, 286, 284, 1, 0, 0, 0, 286, 287, 1, 0, 0, 0, 287, 21, 1, 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 290, 3, 24, 12, 0, 290, 304, 5, 48, 0, 0, 291, 305, 5, 66, 0, 0, 292, 297, 3, 10, 5, 0, 293, 294, 5, 39, 0, 0, 294, 296, 3, 10, 5, 0, 295, 293, 1, 0, 0, 0, 296, 299, 1, 0, 0, 0, 297, 295, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 302, 1, 0, 0, 0, 299, 297, 1, 0, 0, 0, 300, 301, 5, 39, 0, 0, 301, 303, 3, 26, 13, 0, 302, 300, 1, 0, 0, 0, 302, 303, 1, 0, 0, 0, 303, 305, 1, 0, 0, 0, 304, 291, 1, 0, 0, 0, 304, 292, 1, 0, 0, 0, 304, 305, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 307, 5, 55, 0, 0, 307, 23, 1, 0, 0, 0, 308, 309, 3, 76, 38, 0, 309, 25, 1, 0, 0, 0, 310, 311, 5, 69, 0, 0, 311, 316, 3, 28, 14, 0, 312, 313, 5, 39, 0, 0, 313, 315, 3, 28, 14, 0, 314, 312, 1, 0, 0, 0, 315, 318, 1, 0, 0, 0, 316, 314, 1, 0, 0, 0, 316, 317, 1, 0, 0, 0, 317, 319, 1, 0, 0, 0, 318, 316, 1, 0, 0, 0, 319, 320, 5, 70, 0, 0, 320, 27, 1, 0, 0, 0, 321, 322, 3, 110, 55, 0, 322, 323, 5, 38, 0, 0, 323, 324, 3, 72, 36, 0, 324, 29, 1, 0, 0, 0, 325, 326, 3, 68, 34, 0, 326, 31, 1, 0, 0, 0, 327, 328, 5, 12, 0, 0, 328, 329, 3, 34, 17, 0, 329, 33, 1, 0, 0, 0, 330, 335, 3, 36, 18, 0, 331, 332, 5, 39, 0, 0, 332, 334, 3, 36, 18, 0, 333, 331, 1, 0, 0, 0, 334, 337, 1, 0, 0, 0, 335, 333, 1, 0, 0, 0, 335, 336, 1, 0, 0, 0, 336, 35, 1, 0, 0, 0, 337, 335, 1, 0, 0, 0, 338, 339, 3, 62, 31, 0, 339, 340, 5, 36, 0, 0, 340, 342, 1, 0, 0, 0, 341, 338, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 343, 1, 0, 0, 0, 343, 344, 3, 10, 5, 0, 344, 37, 1, 0, 0, 0, 345, 346, 5, 6, 0, 0, 346, 351, 3, 40, 20, 0, 347, 348, 5, 39, 0, 0, 348, 350, 3, 40, 20, 0, 349, 347, 1, 0, 0, 0, 350, 353, 1, 0, 0, 0, 351, 349, 1, 0, 0, 0, 351, 352, 1, 0, 0, 0, 352, 355, 1, 0, 0, 0, 353, 351, 1, 0, 0, 0, 354, 356, 3, 46, 23, 0, 355, 354, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 39, 1, 0, 0, 0, 357, 358, 3, 42, 21, 0, 358, 359, 5, 38, 0, 0, 359, 361, 1, 0, 0, 0, 360, 357, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 362, 1, 0, 0, 0, 362, 363, 3, 44, 22, 0, 363, 41, 1, 0, 0, 0, 364, 365, 5, 83, 0, 0, 365, 43, 1, 0, 0, 0, 366, 367, 7, 2, 0, 0, 367, 45, 1, 0, 0, 0, 368, 371, 3, 48, 24, 0, 369, 371, 3, 50, 25, 0, 370, 368, 1, 0, 0, 0, 370, 369, 1, 0, 0, 0, 371, 47, 1, 0, 0, 0, 372, 373, 5, 82, 0, 0, 373, 378, 5, 83, 0, 0, 374, 375, 5, 39, 0, 0, 375, 377, 5, 83, 0, 0, 376, 374, 1, 0, 0, 0, 377, 380, 1, 0, 0, 0, 378, 376, 1, 0, 0, 0, 378, 379, 1, 0, 0, 0, 379, 49, 1, 0, 0, 0, 380, 378, 1, 0, 0, 0, 381, 382, 5, 72, 0, 0, 382, 383, 3, 48, 24, 0, 383, 384, 5, 73, 0, 0, 384, 51, 1, 0, 0, 0, 385, 386, 5, 19, 0, 0, 386, 391, 3, 40, 20, 0, 387, 388, 5, 39, 0, 0, 388, 390, 3, 40, 20, 0, 389, 387, 1, 0, 0, 0, 390, 393, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 391, 392, 1, 0, 0, 0, 392, 395, 1, 0, 0, 0, 393, 391, 1, 0, 0, 0, 394, 396, 3, 58, 29, 0, 395, 394, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 399, 1, 0, 0, 0, 397, 398, 5, 33, 0, 0, 398, 400, 3, 34, 17, 0, 399, 397, 1, 0, 0, 0, 399, 400, 1, 0, 0, 0, 400, 53, 1, 0, 0, 0, 401, 402, 5, 4, 0, 0, 402, 403, 3, 34, 17, 0, 403, 55, 1, 0, 0, 0, 404, 406, 5, 15, 0, 0, 405, 407, 3, 58, 29, 0, 406, 405, 1, 0, 0, 0, 406, 407, 1, 0, 0, 0, 407, 410, 1, 0, 0, 0, 408, 409, 5, 33, 0, 0, 409, 411, 3, 34, 17, 0, 410, 408, 1, 0, 0, 0, 410, 411, 1, 0, 0, 0, 411, 57, 1, 0, 0, 0, 412, 417, 3, 60, 30, 0, 413, 414, 5, 39, 0, 0, 414, 416, 3, 60, 30, 0, 415, 413, 1, 0, 0, 0, 416, 419, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 417, 418, 1, 0, 0, 0, 418, 59, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 423, 3, 36, 18, 0, 421, 422, 5, 16, 0, 0, 422, 424, 3, 10, 5, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 61, 1, 0, 0, 0, 425, 430, 3, 76, 38, 0, 426, 427, 5, 41, 0, 0, 427, 429, 3, 76, 38, 0, 428, 426, 1, 0, 0, 0, 429, 432, 1, 0, 0, 0, 430, 428, 1, 0, 0, 0, 430, 431, 1, 0, 0, 0, 431, 63, 1, 0, 0, 0, 432, 430, 1, 0, 0, 0, 433, 438, 3, 70, 35, 0, 434, 435, 5, 41, 0, 0, 435, 437, 3, 70, 35, 0, 436, 434, 1, 0, 0, 0, 437, 440, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 438, 439, 1, 0, 0, 0, 439, 65, 1, 0, 0, 0, 440, 438, 1, 0, 0, 0, 441, 446, 3, 64, 32, 0, 442, 443, 5, 39, 0, 0, 443, 445, 3, 64, 32, 0, 444, 442, 1, 0, 0, 0, 445, 448, 1, 0, 0, 0, 446, 444, 1, 0, 0, 0, 446, 447, 1, 0, 0, 0, 447, 67, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 449, 450, 7, 3, 0, 0, 450, 69, 1, 0, 0, 0, 451, 455, 5, 87, 0, 0, 452, 453, 4, 35, 10, 0, 453, 455, 3, 74, 37, 0, 454, 451, 1, 0, 0, 0, 454, 452, 1, 0, 0, 0, 455, 71, 1, 0, 0, 0, 456, 499, 5, 50, 0, 0, 457, 458, 3, 108, 54, 0, 458, 459, 5, 74, 0, 0, 459, 499, 1, 0, 0, 0, 460, 499, 3, 106, 53, 0, 461, 499, 3, 108, 54, 0, 462, 499, 3, 102, 51, 0, 463, 499, 3, 74, 37, 0, 464, 499, 3, 110, 55, 0, 465, 466, 5, 72, 0, 0, 466, 471, 3, 104, 52, 0, 467, 468, 5, 39, 0, 0, 468, 470, 3, 104, 52, 0, 469, 467, 1, 0, 0, 0, 470, 473, 1, 0, 0, 0, 471, 469, 1, 0, 0, 0, 471, 472, 1, 0, 0, 0, 472, 474, 1, 0, 0, 0, 473, 471, 1, 0, 0, 0, 474, 475, 5, 73, 0, 0, 475, 499, 1, 0, 0, 0, 476, 477, 5, 72, 0, 0, 477, 482, 3, 102, 51, 0, 478, 479, 5, 39, 0, 0, 479, 481, 3, 102, 51, 0, 480, 478, 1, 0, 0, 0, 481, 484, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 482, 483, 1, 0, 0, 0, 483, 485, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 486, 5, 73, 0, 0, 486, 499, 1, 0, 0, 0, 487, 488, 5, 72, 0, 0, 488, 493, 3, 110, 55, 0, 489, 490, 5, 39, 0, 0, 490, 492, 3, 110, 55, 0, 491, 489, 1, 0, 0, 0, 492, 495, 1, 0, 0, 0, 493, 491, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 496, 1, 0, 0, 0, 495, 493, 1, 0, 0, 0, 496, 497, 5, 73, 0, 0, 497, 499, 1, 0, 0, 0, 498, 456, 1, 0, 0, 0, 498, 457, 1, 0, 0, 0, 498, 460, 1, 0, 0, 0, 498, 461, 1, 0, 0, 0, 498, 462, 1, 0, 0, 0, 498, 463, 1, 0, 0, 0, 498, 464, 1, 0, 0, 0, 498, 465, 1, 0, 0, 0, 498, 476, 1, 0, 0, 0, 498, 487, 1, 0, 0, 0, 499, 73, 1, 0, 0, 0, 500, 503, 5, 53, 0, 0, 501, 503, 5, 71, 0, 0, 502, 500, 1, 0, 0, 0, 502, 501, 1, 0, 0, 0, 503, 75, 1, 0, 0, 0, 504, 508, 3, 68, 34, 0, 505, 506, 4, 38, 11, 0, 506, 508, 3, 74, 37, 0, 507, 504, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 508, 77, 1, 0, 0, 0, 509, 510, 5, 9, 0, 0, 510, 511, 5, 31, 0, 0, 511, 79, 1, 0, 0, 0, 512, 513, 5, 14, 0, 0, 513, 518, 3, 82, 41, 0, 514, 515, 5, 39, 0, 0, 515, 517, 3, 82, 41, 0, 516, 514, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 516, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 81, 1, 0, 0, 0, 520, 518, 1, 0, 0, 0, 521, 523, 3, 10, 5, 0, 522, 524, 7, 4, 0, 0, 523, 522, 1, 0, 0, 0, 523, 524, 1, 0, 0, 0, 524, 527, 1, 0, 0, 0, 525, 526, 5, 51, 0, 0, 526, 528, 7, 5, 0, 0, 527, 525, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 83, 1, 0, 0, 0, 529, 530, 5, 8, 0, 0, 530, 531, 3, 66, 33, 0, 531, 85, 1, 0, 0, 0, 532, 533, 5, 2, 0, 0, 533, 534, 3, 66, 33, 0, 534, 87, 1, 0, 0, 0, 535, 536, 5, 11, 0, 0, 536, 541, 3, 90, 45, 0, 537, 538, 5, 39, 0, 0, 538, 540, 3, 90, 45, 0, 539, 537, 1, 0, 0, 0, 540, 543, 1, 0, 0, 0, 541, 539, 1, 0, 0, 0, 541, 542, 1, 0, 0, 0, 542, 89, 1, 0, 0, 0, 543, 541, 1, 0, 0, 0, 544, 545, 3, 64, 32, 0, 545, 546, 5, 91, 0, 0, 546, 547, 3, 64, 32, 0, 547, 91, 1, 0, 0, 0, 548, 549, 5, 1, 0, 0, 549, 550, 3, 20, 10, 0, 550, 552, 3, 110, 55, 0, 551, 553, 3, 98, 49, 0, 552, 551, 1, 0, 0, 0, 552, 553, 1, 0, 0, 0, 553, 93, 1, 0, 0, 0, 554, 555, 5, 7, 0, 0, 555, 556, 3, 20, 10, 0, 556, 557, 3, 110, 55, 0, 557, 95, 1, 0, 0, 0, 558, 559, 5, 10, 0, 0, 559, 560, 3, 62, 31, 0, 560, 97, 1, 0, 0, 0, 561, 566, 3, 100, 50, 0, 562, 563, 5, 39, 0, 0, 563, 565, 3, 100, 50, 0, 564, 562, 1, 0, 0, 0, 565, 568, 1, 0, 0, 0, 566, 564, 1, 0, 0, 0, 566, 567, 1, 0, 0, 0, 567, 99, 1, 0, 0, 0, 568, 566, 1, 0, 0, 0, 569, 570, 3, 68, 34, 0, 570, 571, 5, 36, 0, 0, 571, 572, 3, 72, 36, 0, 572, 101, 1, 0, 0, 0, 573, 574, 7, 6, 0, 0, 574, 103, 1, 0, 0, 0, 575, 578, 3, 106, 53, 0, 576, 578, 3, 108, 54, 0, 577, 575, 1, 0, 0, 0, 577, 576, 1, 0, 0, 0, 578, 105, 1, 0, 0, 0, 579, 581, 7, 0, 0, 0, 580, 579, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 1, 0, 0, 0, 582, 583, 5, 32, 0, 0, 583, 107, 1, 0, 0, 0, 584, 586, 7, 0, 0, 0, 585, 584, 1, 0, 0, 0, 585, 586, 1, 0, 0, 0, 586, 587, 1, 0, 0, 0, 587, 588, 5, 31, 0, 0, 588, 109, 1, 0, 0, 0, 589, 590, 5, 30, 0, 0, 590, 111, 1, 0, 0, 0, 591, 592, 7, 7, 0, 0, 592, 113, 1, 0, 0, 0, 593, 594, 5, 5, 0, 0, 594, 595, 3, 116, 58, 0, 595, 115, 1, 0, 0, 0, 596, 597, 5, 72, 0, 0, 597, 598, 3, 2, 1, 0, 598, 599, 5, 73, 0, 0, 599, 117, 1, 0, 0, 0, 600, 601, 5, 13, 0, 0, 601, 602, 5, 107, 0, 0, 602, 119, 1, 0, 0, 0, 603, 604, 5, 3, 0, 0, 604, 607, 5, 97, 0, 0, 605, 606, 5, 95, 0, 0, 606, 608, 3, 64, 32, 0, 607, 605, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 608, 618, 1, 0, 0, 0, 609, 610, 5, 96, 0, 0, 610, 615, 3, 122, 61, 0, 611, 612, 5, 39, 0, 0, 612, 614, 3, 122, 61, 0, 613, 611, 1, 0, 0, 0, 614, 617, 1, 0, 0, 0, 615, 613, 1, 0, 0, 0, 615, 616, 1, 0, 0, 0, 616, 619, 1, 0, 0, 0, 617, 615, 1, 0, 0, 0, 618, 609, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 121, 1, 0, 0, 0, 620, 621, 3, 64, 32, 0, 621, 622, 5, 36, 0, 0, 622, 624, 1, 0, 0, 0, 623, 620, 1, 0, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 1, 0, 0, 0, 625, 626, 3, 64, 32, 0, 626, 123, 1, 0, 0, 0, 627, 628, 5, 18, 0, 0, 628, 629, 3, 40, 20, 0, 629, 630, 5, 95, 0, 0, 630, 631, 3, 66, 33, 0, 631, 125, 1, 0, 0, 0, 632, 633, 5, 17, 0, 0, 633, 636, 3, 58, 29, 0, 634, 635, 5, 33, 0, 0, 635, 637, 3, 34, 17, 0, 636, 634, 1, 0, 0, 0, 636, 637, 1, 0, 0, 0, 637, 127, 1, 0, 0, 0, 638, 640, 7, 8, 0, 0, 639, 638, 1, 0, 0, 0, 639, 640, 1, 0, 0, 0, 640, 641, 1, 0, 0, 0, 641, 642, 5, 20, 0, 0, 642, 643, 3, 130, 65, 0, 643, 644, 3, 132, 66, 0, 644, 129, 1, 0, 0, 0, 645, 648, 3, 40, 20, 0, 646, 647, 5, 91, 0, 0, 647, 649, 3, 68, 34, 0, 648, 646, 1, 0, 0, 0, 648, 649, 1, 0, 0, 0, 649, 131, 1, 0, 0, 0, 650, 651, 5, 95, 0, 0, 651, 656, 3, 134, 67, 0, 652, 653, 5, 39, 0, 0, 653, 655, 3, 134, 67, 0, 654, 652, 1, 0, 0, 0, 655, 658, 1, 0, 0, 0, 656, 654, 1, 0, 0, 0, 656, 657, 1, 0, 0, 0, 657, 133, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 659, 660, 3, 16, 8, 0, 660, 135, 1, 0, 0, 0, 64, 147, 156, 176, 188, 197, 205, 210, 218, 220, 225, 232, 237, 242, 252, 258, 266, 268, 279, 286, 297, 302, 304, 316, 335, 341, 351, 355, 360, 370, 378, 391, 395, 399, 406, 410, 417, 423, 430, 438, 446, 454, 471, 482, 493, 498, 502, 507, 518, 523, 527, 541, 552, 566, 577, 580, 585, 607, 615, 618, 623, 636, 639, 648, 656] \ No newline at end of file +[4, 1, 130, 654, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 146, 8, 1, 10, 1, 12, 1, 149, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 157, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 176, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 188, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 195, 8, 5, 10, 5, 12, 5, 198, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 205, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 210, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 218, 8, 5, 10, 5, 12, 5, 221, 9, 5, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 232, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 237, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 242, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 252, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 258, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 266, 8, 9, 10, 9, 12, 9, 269, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 279, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 284, 8, 10, 10, 10, 12, 10, 287, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 295, 8, 11, 10, 11, 12, 11, 298, 9, 11, 1, 11, 1, 11, 3, 11, 302, 8, 11, 3, 11, 304, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 5, 13, 314, 8, 13, 10, 13, 12, 13, 317, 9, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 5, 17, 333, 8, 17, 10, 17, 12, 17, 336, 9, 17, 1, 18, 1, 18, 1, 18, 3, 18, 341, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 349, 8, 19, 10, 19, 12, 19, 352, 9, 19, 1, 19, 3, 19, 355, 8, 19, 1, 20, 1, 20, 1, 20, 3, 20, 360, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 3, 23, 370, 8, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 376, 8, 24, 10, 24, 12, 24, 379, 9, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 389, 8, 26, 10, 26, 12, 26, 392, 9, 26, 1, 26, 3, 26, 395, 8, 26, 1, 26, 1, 26, 3, 26, 399, 8, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 406, 8, 28, 1, 28, 1, 28, 3, 28, 410, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 415, 8, 29, 10, 29, 12, 29, 418, 9, 29, 1, 30, 1, 30, 1, 30, 3, 30, 423, 8, 30, 1, 31, 1, 31, 1, 31, 5, 31, 428, 8, 31, 10, 31, 12, 31, 431, 9, 31, 1, 32, 1, 32, 1, 32, 5, 32, 436, 8, 32, 10, 32, 12, 32, 439, 9, 32, 1, 33, 1, 33, 1, 33, 5, 33, 444, 8, 33, 10, 33, 12, 33, 447, 9, 33, 1, 34, 1, 34, 1, 35, 1, 35, 3, 35, 453, 8, 35, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 468, 8, 36, 10, 36, 12, 36, 471, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 479, 8, 36, 10, 36, 12, 36, 482, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 490, 8, 36, 10, 36, 12, 36, 493, 9, 36, 1, 36, 1, 36, 3, 36, 497, 8, 36, 1, 37, 1, 37, 3, 37, 501, 8, 37, 1, 38, 1, 38, 3, 38, 505, 8, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 514, 8, 40, 10, 40, 12, 40, 517, 9, 40, 1, 41, 1, 41, 3, 41, 521, 8, 41, 1, 41, 1, 41, 3, 41, 525, 8, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 5, 44, 537, 8, 44, 10, 44, 12, 44, 540, 9, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 3, 46, 550, 8, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 5, 49, 562, 8, 49, 10, 49, 12, 49, 565, 9, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 3, 52, 575, 8, 52, 1, 53, 3, 53, 578, 8, 53, 1, 53, 1, 53, 1, 54, 3, 54, 583, 8, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 3, 60, 605, 8, 60, 1, 60, 1, 60, 1, 60, 1, 60, 5, 60, 611, 8, 60, 10, 60, 12, 60, 614, 9, 60, 3, 60, 616, 8, 60, 1, 61, 1, 61, 1, 61, 3, 61, 621, 8, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 63, 3, 63, 634, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 5, 66, 647, 8, 66, 10, 66, 12, 66, 650, 9, 66, 1, 67, 1, 67, 1, 67, 0, 4, 2, 10, 18, 20, 68, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 0, 9, 1, 0, 63, 64, 1, 0, 65, 67, 2, 0, 29, 29, 82, 82, 1, 0, 73, 74, 2, 0, 34, 34, 39, 39, 2, 0, 42, 42, 45, 45, 2, 0, 41, 41, 55, 55, 2, 0, 56, 56, 58, 62, 2, 0, 17, 17, 22, 23, 678, 0, 136, 1, 0, 0, 0, 2, 139, 1, 0, 0, 0, 4, 156, 1, 0, 0, 0, 6, 175, 1, 0, 0, 0, 8, 177, 1, 0, 0, 0, 10, 209, 1, 0, 0, 0, 12, 236, 1, 0, 0, 0, 14, 238, 1, 0, 0, 0, 16, 251, 1, 0, 0, 0, 18, 257, 1, 0, 0, 0, 20, 278, 1, 0, 0, 0, 22, 288, 1, 0, 0, 0, 24, 307, 1, 0, 0, 0, 26, 309, 1, 0, 0, 0, 28, 320, 1, 0, 0, 0, 30, 324, 1, 0, 0, 0, 32, 326, 1, 0, 0, 0, 34, 329, 1, 0, 0, 0, 36, 340, 1, 0, 0, 0, 38, 344, 1, 0, 0, 0, 40, 359, 1, 0, 0, 0, 42, 363, 1, 0, 0, 0, 44, 365, 1, 0, 0, 0, 46, 369, 1, 0, 0, 0, 48, 371, 1, 0, 0, 0, 50, 380, 1, 0, 0, 0, 52, 384, 1, 0, 0, 0, 54, 400, 1, 0, 0, 0, 56, 403, 1, 0, 0, 0, 58, 411, 1, 0, 0, 0, 60, 419, 1, 0, 0, 0, 62, 424, 1, 0, 0, 0, 64, 432, 1, 0, 0, 0, 66, 440, 1, 0, 0, 0, 68, 448, 1, 0, 0, 0, 70, 452, 1, 0, 0, 0, 72, 496, 1, 0, 0, 0, 74, 500, 1, 0, 0, 0, 76, 504, 1, 0, 0, 0, 78, 506, 1, 0, 0, 0, 80, 509, 1, 0, 0, 0, 82, 518, 1, 0, 0, 0, 84, 526, 1, 0, 0, 0, 86, 529, 1, 0, 0, 0, 88, 532, 1, 0, 0, 0, 90, 541, 1, 0, 0, 0, 92, 545, 1, 0, 0, 0, 94, 551, 1, 0, 0, 0, 96, 555, 1, 0, 0, 0, 98, 558, 1, 0, 0, 0, 100, 566, 1, 0, 0, 0, 102, 570, 1, 0, 0, 0, 104, 574, 1, 0, 0, 0, 106, 577, 1, 0, 0, 0, 108, 582, 1, 0, 0, 0, 110, 586, 1, 0, 0, 0, 112, 588, 1, 0, 0, 0, 114, 590, 1, 0, 0, 0, 116, 593, 1, 0, 0, 0, 118, 597, 1, 0, 0, 0, 120, 600, 1, 0, 0, 0, 122, 620, 1, 0, 0, 0, 124, 624, 1, 0, 0, 0, 126, 629, 1, 0, 0, 0, 128, 635, 1, 0, 0, 0, 130, 640, 1, 0, 0, 0, 132, 642, 1, 0, 0, 0, 134, 651, 1, 0, 0, 0, 136, 137, 3, 2, 1, 0, 137, 138, 5, 0, 0, 1, 138, 1, 1, 0, 0, 0, 139, 140, 6, 1, -1, 0, 140, 141, 3, 4, 2, 0, 141, 147, 1, 0, 0, 0, 142, 143, 10, 1, 0, 0, 143, 144, 5, 28, 0, 0, 144, 146, 3, 6, 3, 0, 145, 142, 1, 0, 0, 0, 146, 149, 1, 0, 0, 0, 147, 145, 1, 0, 0, 0, 147, 148, 1, 0, 0, 0, 148, 3, 1, 0, 0, 0, 149, 147, 1, 0, 0, 0, 150, 157, 3, 114, 57, 0, 151, 157, 3, 38, 19, 0, 152, 157, 3, 32, 16, 0, 153, 157, 3, 118, 59, 0, 154, 155, 4, 2, 1, 0, 155, 157, 3, 52, 26, 0, 156, 150, 1, 0, 0, 0, 156, 151, 1, 0, 0, 0, 156, 152, 1, 0, 0, 0, 156, 153, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 5, 1, 0, 0, 0, 158, 176, 3, 54, 27, 0, 159, 176, 3, 8, 4, 0, 160, 176, 3, 84, 42, 0, 161, 176, 3, 78, 39, 0, 162, 176, 3, 56, 28, 0, 163, 176, 3, 80, 40, 0, 164, 176, 3, 86, 43, 0, 165, 176, 3, 88, 44, 0, 166, 176, 3, 92, 46, 0, 167, 176, 3, 94, 47, 0, 168, 176, 3, 120, 60, 0, 169, 176, 3, 96, 48, 0, 170, 176, 3, 128, 64, 0, 171, 172, 4, 3, 2, 0, 172, 176, 3, 126, 63, 0, 173, 174, 4, 3, 3, 0, 174, 176, 3, 124, 62, 0, 175, 158, 1, 0, 0, 0, 175, 159, 1, 0, 0, 0, 175, 160, 1, 0, 0, 0, 175, 161, 1, 0, 0, 0, 175, 162, 1, 0, 0, 0, 175, 163, 1, 0, 0, 0, 175, 164, 1, 0, 0, 0, 175, 165, 1, 0, 0, 0, 175, 166, 1, 0, 0, 0, 175, 167, 1, 0, 0, 0, 175, 168, 1, 0, 0, 0, 175, 169, 1, 0, 0, 0, 175, 170, 1, 0, 0, 0, 175, 171, 1, 0, 0, 0, 175, 173, 1, 0, 0, 0, 176, 7, 1, 0, 0, 0, 177, 178, 5, 16, 0, 0, 178, 179, 3, 10, 5, 0, 179, 9, 1, 0, 0, 0, 180, 181, 6, 5, -1, 0, 181, 182, 5, 48, 0, 0, 182, 210, 3, 10, 5, 8, 183, 210, 3, 16, 8, 0, 184, 210, 3, 12, 6, 0, 185, 187, 3, 16, 8, 0, 186, 188, 5, 48, 0, 0, 187, 186, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 190, 5, 43, 0, 0, 190, 191, 5, 47, 0, 0, 191, 196, 3, 16, 8, 0, 192, 193, 5, 38, 0, 0, 193, 195, 3, 16, 8, 0, 194, 192, 1, 0, 0, 0, 195, 198, 1, 0, 0, 0, 196, 194, 1, 0, 0, 0, 196, 197, 1, 0, 0, 0, 197, 199, 1, 0, 0, 0, 198, 196, 1, 0, 0, 0, 199, 200, 5, 54, 0, 0, 200, 210, 1, 0, 0, 0, 201, 202, 3, 16, 8, 0, 202, 204, 5, 44, 0, 0, 203, 205, 5, 48, 0, 0, 204, 203, 1, 0, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 1, 0, 0, 0, 206, 207, 5, 49, 0, 0, 207, 210, 1, 0, 0, 0, 208, 210, 3, 14, 7, 0, 209, 180, 1, 0, 0, 0, 209, 183, 1, 0, 0, 0, 209, 184, 1, 0, 0, 0, 209, 185, 1, 0, 0, 0, 209, 201, 1, 0, 0, 0, 209, 208, 1, 0, 0, 0, 210, 219, 1, 0, 0, 0, 211, 212, 10, 5, 0, 0, 212, 213, 5, 33, 0, 0, 213, 218, 3, 10, 5, 6, 214, 215, 10, 4, 0, 0, 215, 216, 5, 51, 0, 0, 216, 218, 3, 10, 5, 5, 217, 211, 1, 0, 0, 0, 217, 214, 1, 0, 0, 0, 218, 221, 1, 0, 0, 0, 219, 217, 1, 0, 0, 0, 219, 220, 1, 0, 0, 0, 220, 11, 1, 0, 0, 0, 221, 219, 1, 0, 0, 0, 222, 224, 3, 16, 8, 0, 223, 225, 5, 48, 0, 0, 224, 223, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 5, 46, 0, 0, 227, 228, 3, 110, 55, 0, 228, 237, 1, 0, 0, 0, 229, 231, 3, 16, 8, 0, 230, 232, 5, 48, 0, 0, 231, 230, 1, 0, 0, 0, 231, 232, 1, 0, 0, 0, 232, 233, 1, 0, 0, 0, 233, 234, 5, 53, 0, 0, 234, 235, 3, 110, 55, 0, 235, 237, 1, 0, 0, 0, 236, 222, 1, 0, 0, 0, 236, 229, 1, 0, 0, 0, 237, 13, 1, 0, 0, 0, 238, 241, 3, 62, 31, 0, 239, 240, 5, 36, 0, 0, 240, 242, 3, 30, 15, 0, 241, 239, 1, 0, 0, 0, 241, 242, 1, 0, 0, 0, 242, 243, 1, 0, 0, 0, 243, 244, 5, 37, 0, 0, 244, 245, 3, 72, 36, 0, 245, 15, 1, 0, 0, 0, 246, 252, 3, 18, 9, 0, 247, 248, 3, 18, 9, 0, 248, 249, 3, 112, 56, 0, 249, 250, 3, 18, 9, 0, 250, 252, 1, 0, 0, 0, 251, 246, 1, 0, 0, 0, 251, 247, 1, 0, 0, 0, 252, 17, 1, 0, 0, 0, 253, 254, 6, 9, -1, 0, 254, 258, 3, 20, 10, 0, 255, 256, 7, 0, 0, 0, 256, 258, 3, 18, 9, 3, 257, 253, 1, 0, 0, 0, 257, 255, 1, 0, 0, 0, 258, 267, 1, 0, 0, 0, 259, 260, 10, 2, 0, 0, 260, 261, 7, 1, 0, 0, 261, 266, 3, 18, 9, 3, 262, 263, 10, 1, 0, 0, 263, 264, 7, 0, 0, 0, 264, 266, 3, 18, 9, 2, 265, 259, 1, 0, 0, 0, 265, 262, 1, 0, 0, 0, 266, 269, 1, 0, 0, 0, 267, 265, 1, 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 19, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 270, 271, 6, 10, -1, 0, 271, 279, 3, 72, 36, 0, 272, 279, 3, 62, 31, 0, 273, 279, 3, 22, 11, 0, 274, 275, 5, 47, 0, 0, 275, 276, 3, 10, 5, 0, 276, 277, 5, 54, 0, 0, 277, 279, 1, 0, 0, 0, 278, 270, 1, 0, 0, 0, 278, 272, 1, 0, 0, 0, 278, 273, 1, 0, 0, 0, 278, 274, 1, 0, 0, 0, 279, 285, 1, 0, 0, 0, 280, 281, 10, 1, 0, 0, 281, 282, 5, 36, 0, 0, 282, 284, 3, 30, 15, 0, 283, 280, 1, 0, 0, 0, 284, 287, 1, 0, 0, 0, 285, 283, 1, 0, 0, 0, 285, 286, 1, 0, 0, 0, 286, 21, 1, 0, 0, 0, 287, 285, 1, 0, 0, 0, 288, 289, 3, 24, 12, 0, 289, 303, 5, 47, 0, 0, 290, 304, 5, 65, 0, 0, 291, 296, 3, 10, 5, 0, 292, 293, 5, 38, 0, 0, 293, 295, 3, 10, 5, 0, 294, 292, 1, 0, 0, 0, 295, 298, 1, 0, 0, 0, 296, 294, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 301, 1, 0, 0, 0, 298, 296, 1, 0, 0, 0, 299, 300, 5, 38, 0, 0, 300, 302, 3, 26, 13, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 304, 1, 0, 0, 0, 303, 290, 1, 0, 0, 0, 303, 291, 1, 0, 0, 0, 303, 304, 1, 0, 0, 0, 304, 305, 1, 0, 0, 0, 305, 306, 5, 54, 0, 0, 306, 23, 1, 0, 0, 0, 307, 308, 3, 76, 38, 0, 308, 25, 1, 0, 0, 0, 309, 310, 5, 68, 0, 0, 310, 315, 3, 28, 14, 0, 311, 312, 5, 38, 0, 0, 312, 314, 3, 28, 14, 0, 313, 311, 1, 0, 0, 0, 314, 317, 1, 0, 0, 0, 315, 313, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 318, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 318, 319, 5, 69, 0, 0, 319, 27, 1, 0, 0, 0, 320, 321, 3, 110, 55, 0, 321, 322, 5, 37, 0, 0, 322, 323, 3, 72, 36, 0, 323, 29, 1, 0, 0, 0, 324, 325, 3, 68, 34, 0, 325, 31, 1, 0, 0, 0, 326, 327, 5, 12, 0, 0, 327, 328, 3, 34, 17, 0, 328, 33, 1, 0, 0, 0, 329, 334, 3, 36, 18, 0, 330, 331, 5, 38, 0, 0, 331, 333, 3, 36, 18, 0, 332, 330, 1, 0, 0, 0, 333, 336, 1, 0, 0, 0, 334, 332, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 35, 1, 0, 0, 0, 336, 334, 1, 0, 0, 0, 337, 338, 3, 62, 31, 0, 338, 339, 5, 35, 0, 0, 339, 341, 1, 0, 0, 0, 340, 337, 1, 0, 0, 0, 340, 341, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 343, 3, 10, 5, 0, 343, 37, 1, 0, 0, 0, 344, 345, 5, 6, 0, 0, 345, 350, 3, 40, 20, 0, 346, 347, 5, 38, 0, 0, 347, 349, 3, 40, 20, 0, 348, 346, 1, 0, 0, 0, 349, 352, 1, 0, 0, 0, 350, 348, 1, 0, 0, 0, 350, 351, 1, 0, 0, 0, 351, 354, 1, 0, 0, 0, 352, 350, 1, 0, 0, 0, 353, 355, 3, 46, 23, 0, 354, 353, 1, 0, 0, 0, 354, 355, 1, 0, 0, 0, 355, 39, 1, 0, 0, 0, 356, 357, 3, 42, 21, 0, 357, 358, 5, 37, 0, 0, 358, 360, 1, 0, 0, 0, 359, 356, 1, 0, 0, 0, 359, 360, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 362, 3, 44, 22, 0, 362, 41, 1, 0, 0, 0, 363, 364, 5, 82, 0, 0, 364, 43, 1, 0, 0, 0, 365, 366, 7, 2, 0, 0, 366, 45, 1, 0, 0, 0, 367, 370, 3, 48, 24, 0, 368, 370, 3, 50, 25, 0, 369, 367, 1, 0, 0, 0, 369, 368, 1, 0, 0, 0, 370, 47, 1, 0, 0, 0, 371, 372, 5, 81, 0, 0, 372, 377, 5, 82, 0, 0, 373, 374, 5, 38, 0, 0, 374, 376, 5, 82, 0, 0, 375, 373, 1, 0, 0, 0, 376, 379, 1, 0, 0, 0, 377, 375, 1, 0, 0, 0, 377, 378, 1, 0, 0, 0, 378, 49, 1, 0, 0, 0, 379, 377, 1, 0, 0, 0, 380, 381, 5, 71, 0, 0, 381, 382, 3, 48, 24, 0, 382, 383, 5, 72, 0, 0, 383, 51, 1, 0, 0, 0, 384, 385, 5, 20, 0, 0, 385, 390, 3, 40, 20, 0, 386, 387, 5, 38, 0, 0, 387, 389, 3, 40, 20, 0, 388, 386, 1, 0, 0, 0, 389, 392, 1, 0, 0, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 394, 1, 0, 0, 0, 392, 390, 1, 0, 0, 0, 393, 395, 3, 58, 29, 0, 394, 393, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 397, 5, 32, 0, 0, 397, 399, 3, 34, 17, 0, 398, 396, 1, 0, 0, 0, 398, 399, 1, 0, 0, 0, 399, 53, 1, 0, 0, 0, 400, 401, 5, 4, 0, 0, 401, 402, 3, 34, 17, 0, 402, 55, 1, 0, 0, 0, 403, 405, 5, 15, 0, 0, 404, 406, 3, 58, 29, 0, 405, 404, 1, 0, 0, 0, 405, 406, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 408, 5, 32, 0, 0, 408, 410, 3, 34, 17, 0, 409, 407, 1, 0, 0, 0, 409, 410, 1, 0, 0, 0, 410, 57, 1, 0, 0, 0, 411, 416, 3, 60, 30, 0, 412, 413, 5, 38, 0, 0, 413, 415, 3, 60, 30, 0, 414, 412, 1, 0, 0, 0, 415, 418, 1, 0, 0, 0, 416, 414, 1, 0, 0, 0, 416, 417, 1, 0, 0, 0, 417, 59, 1, 0, 0, 0, 418, 416, 1, 0, 0, 0, 419, 422, 3, 36, 18, 0, 420, 421, 5, 16, 0, 0, 421, 423, 3, 10, 5, 0, 422, 420, 1, 0, 0, 0, 422, 423, 1, 0, 0, 0, 423, 61, 1, 0, 0, 0, 424, 429, 3, 76, 38, 0, 425, 426, 5, 40, 0, 0, 426, 428, 3, 76, 38, 0, 427, 425, 1, 0, 0, 0, 428, 431, 1, 0, 0, 0, 429, 427, 1, 0, 0, 0, 429, 430, 1, 0, 0, 0, 430, 63, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 437, 3, 70, 35, 0, 433, 434, 5, 40, 0, 0, 434, 436, 3, 70, 35, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 65, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 445, 3, 64, 32, 0, 441, 442, 5, 38, 0, 0, 442, 444, 3, 64, 32, 0, 443, 441, 1, 0, 0, 0, 444, 447, 1, 0, 0, 0, 445, 443, 1, 0, 0, 0, 445, 446, 1, 0, 0, 0, 446, 67, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 448, 449, 7, 3, 0, 0, 449, 69, 1, 0, 0, 0, 450, 453, 5, 86, 0, 0, 451, 453, 3, 74, 37, 0, 452, 450, 1, 0, 0, 0, 452, 451, 1, 0, 0, 0, 453, 71, 1, 0, 0, 0, 454, 497, 5, 49, 0, 0, 455, 456, 3, 108, 54, 0, 456, 457, 5, 73, 0, 0, 457, 497, 1, 0, 0, 0, 458, 497, 3, 106, 53, 0, 459, 497, 3, 108, 54, 0, 460, 497, 3, 102, 51, 0, 461, 497, 3, 74, 37, 0, 462, 497, 3, 110, 55, 0, 463, 464, 5, 71, 0, 0, 464, 469, 3, 104, 52, 0, 465, 466, 5, 38, 0, 0, 466, 468, 3, 104, 52, 0, 467, 465, 1, 0, 0, 0, 468, 471, 1, 0, 0, 0, 469, 467, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 472, 1, 0, 0, 0, 471, 469, 1, 0, 0, 0, 472, 473, 5, 72, 0, 0, 473, 497, 1, 0, 0, 0, 474, 475, 5, 71, 0, 0, 475, 480, 3, 102, 51, 0, 476, 477, 5, 38, 0, 0, 477, 479, 3, 102, 51, 0, 478, 476, 1, 0, 0, 0, 479, 482, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 480, 481, 1, 0, 0, 0, 481, 483, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 483, 484, 5, 72, 0, 0, 484, 497, 1, 0, 0, 0, 485, 486, 5, 71, 0, 0, 486, 491, 3, 110, 55, 0, 487, 488, 5, 38, 0, 0, 488, 490, 3, 110, 55, 0, 489, 487, 1, 0, 0, 0, 490, 493, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 492, 494, 1, 0, 0, 0, 493, 491, 1, 0, 0, 0, 494, 495, 5, 72, 0, 0, 495, 497, 1, 0, 0, 0, 496, 454, 1, 0, 0, 0, 496, 455, 1, 0, 0, 0, 496, 458, 1, 0, 0, 0, 496, 459, 1, 0, 0, 0, 496, 460, 1, 0, 0, 0, 496, 461, 1, 0, 0, 0, 496, 462, 1, 0, 0, 0, 496, 463, 1, 0, 0, 0, 496, 474, 1, 0, 0, 0, 496, 485, 1, 0, 0, 0, 497, 73, 1, 0, 0, 0, 498, 501, 5, 52, 0, 0, 499, 501, 5, 70, 0, 0, 500, 498, 1, 0, 0, 0, 500, 499, 1, 0, 0, 0, 501, 75, 1, 0, 0, 0, 502, 505, 3, 68, 34, 0, 503, 505, 3, 74, 37, 0, 504, 502, 1, 0, 0, 0, 504, 503, 1, 0, 0, 0, 505, 77, 1, 0, 0, 0, 506, 507, 5, 9, 0, 0, 507, 508, 5, 30, 0, 0, 508, 79, 1, 0, 0, 0, 509, 510, 5, 14, 0, 0, 510, 515, 3, 82, 41, 0, 511, 512, 5, 38, 0, 0, 512, 514, 3, 82, 41, 0, 513, 511, 1, 0, 0, 0, 514, 517, 1, 0, 0, 0, 515, 513, 1, 0, 0, 0, 515, 516, 1, 0, 0, 0, 516, 81, 1, 0, 0, 0, 517, 515, 1, 0, 0, 0, 518, 520, 3, 10, 5, 0, 519, 521, 7, 4, 0, 0, 520, 519, 1, 0, 0, 0, 520, 521, 1, 0, 0, 0, 521, 524, 1, 0, 0, 0, 522, 523, 5, 50, 0, 0, 523, 525, 7, 5, 0, 0, 524, 522, 1, 0, 0, 0, 524, 525, 1, 0, 0, 0, 525, 83, 1, 0, 0, 0, 526, 527, 5, 8, 0, 0, 527, 528, 3, 66, 33, 0, 528, 85, 1, 0, 0, 0, 529, 530, 5, 2, 0, 0, 530, 531, 3, 66, 33, 0, 531, 87, 1, 0, 0, 0, 532, 533, 5, 11, 0, 0, 533, 538, 3, 90, 45, 0, 534, 535, 5, 38, 0, 0, 535, 537, 3, 90, 45, 0, 536, 534, 1, 0, 0, 0, 537, 540, 1, 0, 0, 0, 538, 536, 1, 0, 0, 0, 538, 539, 1, 0, 0, 0, 539, 89, 1, 0, 0, 0, 540, 538, 1, 0, 0, 0, 541, 542, 3, 64, 32, 0, 542, 543, 5, 90, 0, 0, 543, 544, 3, 64, 32, 0, 544, 91, 1, 0, 0, 0, 545, 546, 5, 1, 0, 0, 546, 547, 3, 20, 10, 0, 547, 549, 3, 110, 55, 0, 548, 550, 3, 98, 49, 0, 549, 548, 1, 0, 0, 0, 549, 550, 1, 0, 0, 0, 550, 93, 1, 0, 0, 0, 551, 552, 5, 7, 0, 0, 552, 553, 3, 20, 10, 0, 553, 554, 3, 110, 55, 0, 554, 95, 1, 0, 0, 0, 555, 556, 5, 10, 0, 0, 556, 557, 3, 62, 31, 0, 557, 97, 1, 0, 0, 0, 558, 563, 3, 100, 50, 0, 559, 560, 5, 38, 0, 0, 560, 562, 3, 100, 50, 0, 561, 559, 1, 0, 0, 0, 562, 565, 1, 0, 0, 0, 563, 561, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 99, 1, 0, 0, 0, 565, 563, 1, 0, 0, 0, 566, 567, 3, 68, 34, 0, 567, 568, 5, 35, 0, 0, 568, 569, 3, 72, 36, 0, 569, 101, 1, 0, 0, 0, 570, 571, 7, 6, 0, 0, 571, 103, 1, 0, 0, 0, 572, 575, 3, 106, 53, 0, 573, 575, 3, 108, 54, 0, 574, 572, 1, 0, 0, 0, 574, 573, 1, 0, 0, 0, 575, 105, 1, 0, 0, 0, 576, 578, 7, 0, 0, 0, 577, 576, 1, 0, 0, 0, 577, 578, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 580, 5, 31, 0, 0, 580, 107, 1, 0, 0, 0, 581, 583, 7, 0, 0, 0, 582, 581, 1, 0, 0, 0, 582, 583, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 585, 5, 30, 0, 0, 585, 109, 1, 0, 0, 0, 586, 587, 5, 29, 0, 0, 587, 111, 1, 0, 0, 0, 588, 589, 7, 7, 0, 0, 589, 113, 1, 0, 0, 0, 590, 591, 5, 5, 0, 0, 591, 592, 3, 116, 58, 0, 592, 115, 1, 0, 0, 0, 593, 594, 5, 71, 0, 0, 594, 595, 3, 2, 1, 0, 595, 596, 5, 72, 0, 0, 596, 117, 1, 0, 0, 0, 597, 598, 5, 13, 0, 0, 598, 599, 5, 106, 0, 0, 599, 119, 1, 0, 0, 0, 600, 601, 5, 3, 0, 0, 601, 604, 5, 96, 0, 0, 602, 603, 5, 94, 0, 0, 603, 605, 3, 64, 32, 0, 604, 602, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 615, 1, 0, 0, 0, 606, 607, 5, 95, 0, 0, 607, 612, 3, 122, 61, 0, 608, 609, 5, 38, 0, 0, 609, 611, 3, 122, 61, 0, 610, 608, 1, 0, 0, 0, 611, 614, 1, 0, 0, 0, 612, 610, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 616, 1, 0, 0, 0, 614, 612, 1, 0, 0, 0, 615, 606, 1, 0, 0, 0, 615, 616, 1, 0, 0, 0, 616, 121, 1, 0, 0, 0, 617, 618, 3, 64, 32, 0, 618, 619, 5, 35, 0, 0, 619, 621, 1, 0, 0, 0, 620, 617, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 3, 64, 32, 0, 623, 123, 1, 0, 0, 0, 624, 625, 5, 19, 0, 0, 625, 626, 3, 40, 20, 0, 626, 627, 5, 94, 0, 0, 627, 628, 3, 66, 33, 0, 628, 125, 1, 0, 0, 0, 629, 630, 5, 18, 0, 0, 630, 633, 3, 58, 29, 0, 631, 632, 5, 32, 0, 0, 632, 634, 3, 34, 17, 0, 633, 631, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 127, 1, 0, 0, 0, 635, 636, 7, 8, 0, 0, 636, 637, 5, 120, 0, 0, 637, 638, 3, 130, 65, 0, 638, 639, 3, 132, 66, 0, 639, 129, 1, 0, 0, 0, 640, 641, 3, 40, 20, 0, 641, 131, 1, 0, 0, 0, 642, 643, 5, 94, 0, 0, 643, 648, 3, 134, 67, 0, 644, 645, 5, 38, 0, 0, 645, 647, 3, 134, 67, 0, 646, 644, 1, 0, 0, 0, 647, 650, 1, 0, 0, 0, 648, 646, 1, 0, 0, 0, 648, 649, 1, 0, 0, 0, 649, 133, 1, 0, 0, 0, 650, 648, 1, 0, 0, 0, 651, 652, 3, 16, 8, 0, 652, 135, 1, 0, 0, 0, 62, 147, 156, 175, 187, 196, 204, 209, 217, 219, 224, 231, 236, 241, 251, 257, 265, 267, 278, 285, 296, 301, 303, 315, 334, 340, 350, 354, 359, 369, 377, 390, 394, 398, 405, 409, 416, 422, 429, 437, 445, 452, 469, 480, 491, 496, 500, 504, 515, 520, 524, 538, 549, 563, 574, 577, 582, 604, 612, 615, 620, 633, 648] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index c6d97d4a78511..098b4c12b3cd1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -27,31 +27,31 @@ public class EsqlBaseParser extends ParserConfig { public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, DEV_JOIN=20, - DEV_JOIN_FULL=21, DEV_JOIN_LEFT=22, DEV_JOIN_RIGHT=23, DEV_JOIN_LOOKUP=24, - UNKNOWN_CMD=25, LINE_COMMENT=26, MULTILINE_COMMENT=27, WS=28, PIPE=29, - QUOTED_STRING=30, INTEGER_LITERAL=31, DECIMAL_LITERAL=32, BY=33, AND=34, - ASC=35, ASSIGN=36, CAST_OP=37, COLON=38, COMMA=39, DESC=40, DOT=41, FALSE=42, - FIRST=43, IN=44, IS=45, LAST=46, LIKE=47, LP=48, NOT=49, NULL=50, NULLS=51, - OR=52, PARAM=53, RLIKE=54, RP=55, TRUE=56, EQ=57, CIEQ=58, NEQ=59, LT=60, - LTE=61, GT=62, GTE=63, PLUS=64, MINUS=65, ASTERISK=66, SLASH=67, PERCENT=68, - LEFT_BRACES=69, RIGHT_BRACES=70, NAMED_OR_POSITIONAL_PARAM=71, OPENING_BRACKET=72, - CLOSING_BRACKET=73, UNQUOTED_IDENTIFIER=74, QUOTED_IDENTIFIER=75, EXPR_LINE_COMMENT=76, - EXPR_MULTILINE_COMMENT=77, EXPR_WS=78, EXPLAIN_WS=79, EXPLAIN_LINE_COMMENT=80, - EXPLAIN_MULTILINE_COMMENT=81, METADATA=82, UNQUOTED_SOURCE=83, FROM_LINE_COMMENT=84, - FROM_MULTILINE_COMMENT=85, FROM_WS=86, ID_PATTERN=87, PROJECT_LINE_COMMENT=88, - PROJECT_MULTILINE_COMMENT=89, PROJECT_WS=90, AS=91, RENAME_LINE_COMMENT=92, - RENAME_MULTILINE_COMMENT=93, RENAME_WS=94, ON=95, WITH=96, ENRICH_POLICY_NAME=97, - ENRICH_LINE_COMMENT=98, ENRICH_MULTILINE_COMMENT=99, ENRICH_WS=100, ENRICH_FIELD_LINE_COMMENT=101, - ENRICH_FIELD_MULTILINE_COMMENT=102, ENRICH_FIELD_WS=103, MVEXPAND_LINE_COMMENT=104, - MVEXPAND_MULTILINE_COMMENT=105, MVEXPAND_WS=106, INFO=107, SHOW_LINE_COMMENT=108, - SHOW_MULTILINE_COMMENT=109, SHOW_WS=110, SETTING=111, SETTING_LINE_COMMENT=112, - SETTTING_MULTILINE_COMMENT=113, SETTING_WS=114, LOOKUP_LINE_COMMENT=115, - LOOKUP_MULTILINE_COMMENT=116, LOOKUP_WS=117, LOOKUP_FIELD_LINE_COMMENT=118, - LOOKUP_FIELD_MULTILINE_COMMENT=119, LOOKUP_FIELD_WS=120, USING=121, JOIN_LINE_COMMENT=122, - JOIN_MULTILINE_COMMENT=123, JOIN_WS=124, METRICS_LINE_COMMENT=125, METRICS_MULTILINE_COMMENT=126, - METRICS_WS=127, CLOSING_METRICS_LINE_COMMENT=128, CLOSING_METRICS_MULTILINE_COMMENT=129, - CLOSING_METRICS_WS=130; + WHERE=16, JOIN_LOOKUP=17, DEV_INLINESTATS=18, DEV_LOOKUP=19, DEV_METRICS=20, + DEV_JOIN_FULL=21, DEV_JOIN_LEFT=22, DEV_JOIN_RIGHT=23, UNKNOWN_CMD=24, + LINE_COMMENT=25, MULTILINE_COMMENT=26, WS=27, PIPE=28, QUOTED_STRING=29, + INTEGER_LITERAL=30, DECIMAL_LITERAL=31, BY=32, AND=33, ASC=34, ASSIGN=35, + CAST_OP=36, COLON=37, COMMA=38, DESC=39, DOT=40, FALSE=41, FIRST=42, IN=43, + IS=44, LAST=45, LIKE=46, LP=47, NOT=48, NULL=49, NULLS=50, OR=51, PARAM=52, + RLIKE=53, RP=54, TRUE=55, EQ=56, CIEQ=57, NEQ=58, LT=59, LTE=60, GT=61, + GTE=62, PLUS=63, MINUS=64, ASTERISK=65, SLASH=66, PERCENT=67, LEFT_BRACES=68, + RIGHT_BRACES=69, NAMED_OR_POSITIONAL_PARAM=70, OPENING_BRACKET=71, CLOSING_BRACKET=72, + UNQUOTED_IDENTIFIER=73, QUOTED_IDENTIFIER=74, EXPR_LINE_COMMENT=75, EXPR_MULTILINE_COMMENT=76, + EXPR_WS=77, EXPLAIN_WS=78, EXPLAIN_LINE_COMMENT=79, EXPLAIN_MULTILINE_COMMENT=80, + METADATA=81, UNQUOTED_SOURCE=82, FROM_LINE_COMMENT=83, FROM_MULTILINE_COMMENT=84, + FROM_WS=85, ID_PATTERN=86, PROJECT_LINE_COMMENT=87, PROJECT_MULTILINE_COMMENT=88, + PROJECT_WS=89, AS=90, RENAME_LINE_COMMENT=91, RENAME_MULTILINE_COMMENT=92, + RENAME_WS=93, ON=94, WITH=95, ENRICH_POLICY_NAME=96, ENRICH_LINE_COMMENT=97, + ENRICH_MULTILINE_COMMENT=98, ENRICH_WS=99, ENRICH_FIELD_LINE_COMMENT=100, + ENRICH_FIELD_MULTILINE_COMMENT=101, ENRICH_FIELD_WS=102, MVEXPAND_LINE_COMMENT=103, + MVEXPAND_MULTILINE_COMMENT=104, MVEXPAND_WS=105, INFO=106, SHOW_LINE_COMMENT=107, + SHOW_MULTILINE_COMMENT=108, SHOW_WS=109, SETTING=110, SETTING_LINE_COMMENT=111, + SETTTING_MULTILINE_COMMENT=112, SETTING_WS=113, LOOKUP_LINE_COMMENT=114, + LOOKUP_MULTILINE_COMMENT=115, LOOKUP_WS=116, LOOKUP_FIELD_LINE_COMMENT=117, + LOOKUP_FIELD_MULTILINE_COMMENT=118, LOOKUP_FIELD_WS=119, JOIN=120, USING=121, + JOIN_LINE_COMMENT=122, JOIN_MULTILINE_COMMENT=123, JOIN_WS=124, METRICS_LINE_COMMENT=125, + METRICS_MULTILINE_COMMENT=126, METRICS_WS=127, CLOSING_METRICS_LINE_COMMENT=128, + CLOSING_METRICS_MULTILINE_COMMENT=129, CLOSING_METRICS_WS=130; public static final int RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, @@ -100,7 +100,7 @@ private static String[] makeLiteralNames() { return new String[] { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", - "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, + "'sort'", "'stats'", "'where'", "'lookup'", null, null, null, null, null, null, null, null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "':'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", @@ -110,7 +110,7 @@ private static String[] makeLiteralNames() { null, null, null, null, null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, null, null, null, null, null, null, null, "'info'", null, null, null, null, null, null, null, null, null, - null, null, null, null, "'USING'" + null, null, null, null, "'join'", "'USING'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -118,13 +118,13 @@ private static String[] makeSymbolicNames() { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "DEV_JOIN", - "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "DEV_JOIN_LOOKUP", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", - "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", - "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "WHERE", "JOIN_LOOKUP", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", + "DEV_JOIN_FULL", "DEV_JOIN_LEFT", "DEV_JOIN_RIGHT", "UNKNOWN_CMD", "LINE_COMMENT", + "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", + "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", + "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", + "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", @@ -139,8 +139,8 @@ private static String[] makeSymbolicNames() { "SHOW_WS", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", - "USING", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", "METRICS_LINE_COMMENT", - "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", + "JOIN", "USING", "JOIN_LINE_COMMENT", "JOIN_MULTILINE_COMMENT", "JOIN_WS", + "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS" }; } @@ -499,15 +499,15 @@ public EnrichCommandContext enrichCommand() { public MvExpandCommandContext mvExpandCommand() { return getRuleContext(MvExpandCommandContext.class,0); } + public JoinCommandContext joinCommand() { + return getRuleContext(JoinCommandContext.class,0); + } public InlinestatsCommandContext inlinestatsCommand() { return getRuleContext(InlinestatsCommandContext.class,0); } public LookupCommandContext lookupCommand() { return getRuleContext(LookupCommandContext.class,0); } - public JoinCommandContext joinCommand() { - return getRuleContext(JoinCommandContext.class,0); - } @SuppressWarnings("this-escape") public ProcessingCommandContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -532,7 +532,7 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(176); + setState(175); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: @@ -623,27 +623,25 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce enterOuterAlt(_localctx, 13); { setState(170); - if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(171); - inlinestatsCommand(); + joinCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(172); + setState(171); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(173); - lookupCommand(); + setState(172); + inlinestatsCommand(); } break; case 15: enterOuterAlt(_localctx, 15); { - setState(174); + setState(173); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(175); - joinCommand(); + setState(174); + lookupCommand(); } break; } @@ -691,9 +689,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(178); + setState(177); match(WHERE); - setState(179); + setState(178); booleanExpression(0); } } @@ -909,7 +907,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(210); + setState(209); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -918,9 +916,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(182); + setState(181); match(NOT); - setState(183); + setState(182); booleanExpression(8); } break; @@ -929,7 +927,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(184); + setState(183); valueExpression(); } break; @@ -938,7 +936,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(185); + setState(184); regexBooleanExpression(); } break; @@ -947,41 +945,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(186); + setState(185); valueExpression(); - setState(188); + setState(187); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(187); + setState(186); match(NOT); } } - setState(190); + setState(189); match(IN); - setState(191); + setState(190); match(LP); - setState(192); + setState(191); valueExpression(); - setState(197); + setState(196); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(193); + setState(192); match(COMMA); - setState(194); + setState(193); valueExpression(); } } - setState(199); + setState(198); _errHandler.sync(this); _la = _input.LA(1); } - setState(200); + setState(199); match(RP); } break; @@ -990,21 +988,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(202); + setState(201); valueExpression(); - setState(203); + setState(202); match(IS); - setState(205); + setState(204); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(204); + setState(203); match(NOT); } } - setState(207); + setState(206); match(NULL); } break; @@ -1013,13 +1011,13 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(209); + setState(208); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(220); + setState(219); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1027,7 +1025,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(218); + setState(217); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1035,11 +1033,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(212); + setState(211); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(213); + setState(212); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(214); + setState(213); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1048,18 +1046,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(215); + setState(214); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(216); + setState(215); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(217); + setState(216); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(222); + setState(221); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1114,48 +1112,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(237); + setState(236); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(223); + setState(222); valueExpression(); - setState(225); + setState(224); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(224); + setState(223); match(NOT); } } - setState(227); + setState(226); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(228); + setState(227); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(230); + setState(229); valueExpression(); - setState(232); + setState(231); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(231); + setState(230); match(NOT); } } - setState(234); + setState(233); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(235); + setState(234); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1215,23 +1213,23 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(239); + setState(238); ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName(); - setState(242); + setState(241); _errHandler.sync(this); _la = _input.LA(1); if (_la==CAST_OP) { { - setState(240); + setState(239); match(CAST_OP); - setState(241); + setState(240); ((MatchBooleanExpressionContext)_localctx).fieldType = dataType(); } } - setState(244); + setState(243); match(COLON); - setState(245); + setState(244); ((MatchBooleanExpressionContext)_localctx).matchQuery = constant(); } } @@ -1315,14 +1313,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(252); + setState(251); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(247); + setState(246); operatorExpression(0); } break; @@ -1330,11 +1328,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(248); + setState(247); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(249); + setState(248); comparisonOperator(); - setState(250); + setState(249); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1459,7 +1457,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(258); + setState(257); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1468,7 +1466,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(255); + setState(254); primaryExpression(0); } break; @@ -1477,7 +1475,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(256); + setState(255); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1488,13 +1486,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(257); + setState(256); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(268); + setState(267); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1502,7 +1500,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(266); + setState(265); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: @@ -1510,12 +1508,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(260); + setState(259); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(261); + setState(260); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { + if ( !(((((_la - 65)) & ~0x3f) == 0 && ((1L << (_la - 65)) & 7L) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1523,7 +1521,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(262); + setState(261); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1532,9 +1530,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(263); + setState(262); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(264); + setState(263); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1545,14 +1543,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(265); + setState(264); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(270); + setState(269); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); } @@ -1710,7 +1708,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(279); + setState(278); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) { case 1: @@ -1719,7 +1717,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(272); + setState(271); constant(); } break; @@ -1728,7 +1726,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(273); + setState(272); qualifiedName(); } break; @@ -1737,7 +1735,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(274); + setState(273); functionExpression(); } break; @@ -1746,17 +1744,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(275); + setState(274); match(LP); - setState(276); + setState(275); booleanExpression(0); - setState(277); + setState(276); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(286); + setState(285); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1767,16 +1765,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(281); + setState(280); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(282); + setState(281); match(CAST_OP); - setState(283); + setState(282); dataType(); } } } - setState(288); + setState(287); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); } @@ -1842,50 +1840,64 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx int _alt; enterOuterAlt(_localctx, 1); { - setState(289); + setState(288); functionName(); - setState(290); + setState(289); match(LP); - setState(304); + setState(303); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) { - case 1: + switch (_input.LA(1)) { + case ASTERISK: { - setState(291); + setState(290); match(ASTERISK); } break; - case 2: + case QUOTED_STRING: + case INTEGER_LITERAL: + case DECIMAL_LITERAL: + case FALSE: + case LP: + case NOT: + case NULL: + case PARAM: + case TRUE: + case PLUS: + case MINUS: + case NAMED_OR_POSITIONAL_PARAM: + case OPENING_BRACKET: + case UNQUOTED_IDENTIFIER: + case QUOTED_IDENTIFIER: { { - setState(292); + setState(291); booleanExpression(0); - setState(297); + setState(296); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(293); + setState(292); match(COMMA); - setState(294); + setState(293); booleanExpression(0); } } } - setState(299); + setState(298); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); } - setState(302); + setState(301); _errHandler.sync(this); _la = _input.LA(1); if (_la==COMMA) { { - setState(300); + setState(299); match(COMMA); - setState(301); + setState(300); mapExpression(); } } @@ -1893,8 +1905,12 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } } break; + case RP: + break; + default: + break; } - setState(306); + setState(305); match(RP); } } @@ -1940,7 +1956,7 @@ public final FunctionNameContext functionName() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(308); + setState(307); identifierOrParameter(); } } @@ -1996,27 +2012,27 @@ public final MapExpressionContext mapExpression() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(310); + setState(309); match(LEFT_BRACES); - setState(311); + setState(310); entryExpression(); - setState(316); + setState(315); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(312); + setState(311); match(COMMA); - setState(313); + setState(312); entryExpression(); } } - setState(318); + setState(317); _errHandler.sync(this); _la = _input.LA(1); } - setState(319); + setState(318); match(RIGHT_BRACES); } } @@ -2068,11 +2084,11 @@ public final EntryExpressionContext entryExpression() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(321); + setState(320); ((EntryExpressionContext)_localctx).key = string(); - setState(322); + setState(321); match(COLON); - setState(323); + setState(322); ((EntryExpressionContext)_localctx).value = constant(); } } @@ -2130,7 +2146,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(325); + setState(324); identifier(); } } @@ -2177,9 +2193,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(327); + setState(326); match(ROW); - setState(328); + setState(327); fields(); } } @@ -2233,23 +2249,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(330); + setState(329); field(); - setState(335); + setState(334); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(331); + setState(330); match(COMMA); - setState(332); + setState(331); field(); } } } - setState(337); + setState(336); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } @@ -2301,19 +2317,19 @@ public final FieldContext field() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(341); + setState(340); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(338); + setState(337); qualifiedName(); - setState(339); + setState(338); match(ASSIGN); } break; } - setState(343); + setState(342); booleanExpression(0); } } @@ -2371,34 +2387,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(345); + setState(344); match(FROM); - setState(346); + setState(345); indexPattern(); - setState(351); + setState(350); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,25,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(347); + setState(346); match(COMMA); - setState(348); + setState(347); indexPattern(); } } } - setState(353); + setState(352); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,25,_ctx); } - setState(355); + setState(354); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { case 1: { - setState(354); + setState(353); metadata(); } break; @@ -2451,19 +2467,19 @@ public final IndexPatternContext indexPattern() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(360); + setState(359); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) { case 1: { - setState(357); + setState(356); clusterString(); - setState(358); + setState(357); match(COLON); } break; } - setState(362); + setState(361); indexString(); } } @@ -2507,7 +2523,7 @@ public final ClusterStringContext clusterString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(364); + setState(363); match(UNQUOTED_SOURCE); } } @@ -2553,7 +2569,7 @@ public final IndexStringContext indexString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(366); + setState(365); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2608,20 +2624,20 @@ public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 46, RULE_metadata); try { - setState(370); + setState(369); _errHandler.sync(this); switch (_input.LA(1)) { case METADATA: enterOuterAlt(_localctx, 1); { - setState(368); + setState(367); metadataOption(); } break; case OPENING_BRACKET: enterOuterAlt(_localctx, 2); { - setState(369); + setState(368); deprecated_metadata(); } break; @@ -2678,25 +2694,25 @@ public final MetadataOptionContext metadataOption() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(372); + setState(371); match(METADATA); - setState(373); + setState(372); match(UNQUOTED_SOURCE); - setState(378); + setState(377); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,29,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(374); + setState(373); match(COMMA); - setState(375); + setState(374); match(UNQUOTED_SOURCE); } } } - setState(380); + setState(379); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,29,_ctx); } @@ -2745,11 +2761,11 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition try { enterOuterAlt(_localctx, 1); { - setState(381); + setState(380); match(OPENING_BRACKET); - setState(382); + setState(381); metadataOption(); - setState(383); + setState(382); match(CLOSING_BRACKET); } } @@ -2813,46 +2829,46 @@ public final MetricsCommandContext metricsCommand() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(385); + setState(384); match(DEV_METRICS); - setState(386); + setState(385); indexPattern(); - setState(391); + setState(390); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,30,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(387); + setState(386); match(COMMA); - setState(388); + setState(387); indexPattern(); } } } - setState(393); + setState(392); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,30,_ctx); } - setState(395); + setState(394); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(394); + setState(393); ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(399); + setState(398); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { case 1: { - setState(397); + setState(396); match(BY); - setState(398); + setState(397); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2902,9 +2918,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(401); + setState(400); match(EVAL); - setState(402); + setState(401); fields(); } } @@ -2957,26 +2973,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(404); + setState(403); match(STATS); - setState(406); + setState(405); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: { - setState(405); + setState(404); ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(410); + setState(409); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { case 1: { - setState(408); + setState(407); match(BY); - setState(409); + setState(408); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -3033,23 +3049,23 @@ public final AggFieldsContext aggFields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(412); + setState(411); aggField(); - setState(417); + setState(416); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(413); + setState(412); match(COMMA); - setState(414); + setState(413); aggField(); } } } - setState(419); + setState(418); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } @@ -3101,16 +3117,16 @@ public final AggFieldContext aggField() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(420); + setState(419); field(); - setState(423); + setState(422); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { case 1: { - setState(421); + setState(420); match(WHERE); - setState(422); + setState(421); booleanExpression(0); } break; @@ -3167,23 +3183,23 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(425); + setState(424); identifierOrParameter(); - setState(430); + setState(429); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(426); + setState(425); match(DOT); - setState(427); + setState(426); identifierOrParameter(); } } } - setState(432); + setState(431); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } @@ -3239,23 +3255,23 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti int _alt; enterOuterAlt(_localctx, 1); { - setState(433); + setState(432); identifierPattern(); - setState(438); + setState(437); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,38,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(434); + setState(433); match(DOT); - setState(435); + setState(434); identifierPattern(); } } } - setState(440); + setState(439); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,38,_ctx); } @@ -3311,23 +3327,23 @@ public final QualifiedNamePatternsContext qualifiedNamePatterns() throws Recogni int _alt; enterOuterAlt(_localctx, 1); { - setState(441); + setState(440); qualifiedNamePattern(); - setState(446); + setState(445); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,39,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(442); + setState(441); match(COMMA); - setState(443); + setState(442); qualifiedNamePattern(); } } } - setState(448); + setState(447); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,39,_ctx); } @@ -3375,7 +3391,7 @@ public final IdentifierContext identifier() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(449); + setState(448); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3428,25 +3444,26 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); enterRule(_localctx, 70, RULE_identifierPattern); try { - setState(454); + setState(452); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) { - case 1: + switch (_input.LA(1)) { + case ID_PATTERN: enterOuterAlt(_localctx, 1); { - setState(451); + setState(450); match(ID_PATTERN); } break; - case 2: + case PARAM: + case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(452); - if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(453); + setState(451); parameter(); } break; + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -3716,14 +3733,14 @@ public final ConstantContext constant() throws RecognitionException { enterRule(_localctx, 72, RULE_constant); int _la; try { - setState(498); + setState(496); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(456); + setState(454); match(NULL); } break; @@ -3731,9 +3748,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(457); + setState(455); integerValue(); - setState(458); + setState(456); match(UNQUOTED_IDENTIFIER); } break; @@ -3741,7 +3758,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(460); + setState(458); decimalValue(); } break; @@ -3749,7 +3766,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(461); + setState(459); integerValue(); } break; @@ -3757,7 +3774,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(462); + setState(460); booleanValue(); } break; @@ -3765,7 +3782,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(463); + setState(461); parameter(); } break; @@ -3773,7 +3790,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(464); + setState(462); string(); } break; @@ -3781,27 +3798,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(465); + setState(463); match(OPENING_BRACKET); - setState(466); + setState(464); numericValue(); - setState(471); + setState(469); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(467); + setState(465); match(COMMA); - setState(468); + setState(466); numericValue(); } } - setState(473); + setState(471); _errHandler.sync(this); _la = _input.LA(1); } - setState(474); + setState(472); match(CLOSING_BRACKET); } break; @@ -3809,27 +3826,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(476); + setState(474); match(OPENING_BRACKET); - setState(477); + setState(475); booleanValue(); - setState(482); + setState(480); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(478); + setState(476); match(COMMA); - setState(479); + setState(477); booleanValue(); } } - setState(484); + setState(482); _errHandler.sync(this); _la = _input.LA(1); } - setState(485); + setState(483); match(CLOSING_BRACKET); } break; @@ -3837,27 +3854,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(487); + setState(485); match(OPENING_BRACKET); - setState(488); + setState(486); string(); - setState(493); + setState(491); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(489); + setState(487); match(COMMA); - setState(490); + setState(488); string(); } } - setState(495); + setState(493); _errHandler.sync(this); _la = _input.LA(1); } - setState(496); + setState(494); match(CLOSING_BRACKET); } break; @@ -3931,14 +3948,14 @@ public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); enterRule(_localctx, 74, RULE_parameter); try { - setState(502); + setState(500); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(500); + setState(498); match(PARAM); } break; @@ -3946,7 +3963,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(501); + setState(499); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3997,25 +4014,27 @@ public final IdentifierOrParameterContext identifierOrParameter() throws Recogni IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); enterRule(_localctx, 76, RULE_identifierOrParameter); try { - setState(507); + setState(504); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { - case 1: + switch (_input.LA(1)) { + case UNQUOTED_IDENTIFIER: + case QUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(504); + setState(502); identifier(); } break; - case 2: + case PARAM: + case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(505); - if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(506); + setState(503); parameter(); } break; + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -4059,9 +4078,9 @@ public final LimitCommandContext limitCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(509); + setState(506); match(LIMIT); - setState(510); + setState(507); match(INTEGER_LITERAL); } } @@ -4116,25 +4135,25 @@ public final SortCommandContext sortCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(512); + setState(509); match(SORT); - setState(513); + setState(510); orderExpression(); - setState(518); + setState(515); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,47,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(514); + setState(511); match(COMMA); - setState(515); + setState(512); orderExpression(); } } } - setState(520); + setState(517); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,47,_ctx); } @@ -4190,14 +4209,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(518); booleanExpression(0); - setState(523); + setState(520); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(522); + setState(519); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4211,14 +4230,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(527); + setState(524); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: { - setState(525); + setState(522); match(NULLS); - setState(526); + setState(523); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4277,9 +4296,9 @@ public final KeepCommandContext keepCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(529); + setState(526); match(KEEP); - setState(530); + setState(527); qualifiedNamePatterns(); } } @@ -4326,9 +4345,9 @@ public final DropCommandContext dropCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(532); + setState(529); match(DROP); - setState(533); + setState(530); qualifiedNamePatterns(); } } @@ -4383,25 +4402,25 @@ public final RenameCommandContext renameCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(535); + setState(532); match(RENAME); - setState(536); + setState(533); renameClause(); - setState(541); + setState(538); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,50,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(537); + setState(534); match(COMMA); - setState(538); + setState(535); renameClause(); } } } - setState(543); + setState(540); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,50,_ctx); } @@ -4455,11 +4474,11 @@ public final RenameClauseContext renameClause() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(544); + setState(541); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(545); + setState(542); match(AS); - setState(546); + setState(543); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4512,18 +4531,18 @@ public final DissectCommandContext dissectCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(548); + setState(545); match(DISSECT); - setState(549); + setState(546); primaryExpression(0); - setState(550); + setState(547); string(); - setState(552); + setState(549); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: { - setState(551); + setState(548); commandOptions(); } break; @@ -4576,11 +4595,11 @@ public final GrokCommandContext grokCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(554); + setState(551); match(GROK); - setState(555); + setState(552); primaryExpression(0); - setState(556); + setState(553); string(); } } @@ -4627,9 +4646,9 @@ public final MvExpandCommandContext mvExpandCommand() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(558); + setState(555); match(MV_EXPAND); - setState(559); + setState(556); qualifiedName(); } } @@ -4683,23 +4702,23 @@ public final CommandOptionsContext commandOptions() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(561); + setState(558); commandOption(); - setState(566); + setState(563); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,52,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(562); + setState(559); match(COMMA); - setState(563); + setState(560); commandOption(); } } } - setState(568); + setState(565); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,52,_ctx); } @@ -4751,11 +4770,11 @@ public final CommandOptionContext commandOption() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(569); + setState(566); identifier(); - setState(570); + setState(567); match(ASSIGN); - setState(571); + setState(568); constant(); } } @@ -4801,7 +4820,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(573); + setState(570); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4856,20 +4875,20 @@ public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); enterRule(_localctx, 104, RULE_numericValue); try { - setState(577); + setState(574); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(575); + setState(572); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(576); + setState(573); integerValue(); } break; @@ -4918,12 +4937,12 @@ public final DecimalValueContext decimalValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(580); + setState(577); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(579); + setState(576); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4936,7 +4955,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(582); + setState(579); match(DECIMAL_LITERAL); } } @@ -4983,12 +5002,12 @@ public final IntegerValueContext integerValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(585); + setState(582); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(584); + setState(581); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -5001,7 +5020,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(587); + setState(584); match(INTEGER_LITERAL); } } @@ -5045,7 +5064,7 @@ public final StringContext string() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(589); + setState(586); match(QUOTED_STRING); } } @@ -5095,9 +5114,9 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(591); + setState(588); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 9007199254740992000L) != 0)) ) { _errHandler.recoverInline(this); } else { @@ -5150,9 +5169,9 @@ public final ExplainCommandContext explainCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(593); + setState(590); match(EXPLAIN); - setState(594); + setState(591); subqueryExpression(); } } @@ -5200,11 +5219,11 @@ public final SubqueryExpressionContext subqueryExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(596); + setState(593); match(OPENING_BRACKET); - setState(597); + setState(594); query(0); - setState(598); + setState(595); match(CLOSING_BRACKET); } } @@ -5261,9 +5280,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(600); + setState(597); match(SHOW); - setState(601); + setState(598); match(INFO); } } @@ -5326,46 +5345,46 @@ public final EnrichCommandContext enrichCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(603); + setState(600); match(ENRICH); - setState(604); + setState(601); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(607); + setState(604); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(605); + setState(602); match(ON); - setState(606); + setState(603); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(618); + setState(615); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { case 1: { - setState(609); + setState(606); match(WITH); - setState(610); + setState(607); enrichWithClause(); - setState(615); + setState(612); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,57,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(611); + setState(608); match(COMMA); - setState(612); + setState(609); enrichWithClause(); } } } - setState(617); + setState(614); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,57,_ctx); } @@ -5422,19 +5441,19 @@ public final EnrichWithClauseContext enrichWithClause() throws RecognitionExcept try { enterOuterAlt(_localctx, 1); { - setState(623); + setState(620); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,59,_ctx) ) { case 1: { - setState(620); + setState(617); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(621); + setState(618); match(ASSIGN); } break; } - setState(625); + setState(622); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5487,13 +5506,13 @@ public final LookupCommandContext lookupCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(627); + setState(624); match(DEV_LOOKUP); - setState(628); + setState(625); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(629); + setState(626); match(ON); - setState(630); + setState(627); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5546,18 +5565,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(632); + setState(629); match(DEV_INLINESTATS); - setState(633); + setState(630); ((InlinestatsCommandContext)_localctx).stats = aggFields(); - setState(636); + setState(633); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,60,_ctx) ) { case 1: { - setState(634); + setState(631); match(BY); - setState(635); + setState(632); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5578,14 +5597,14 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx @SuppressWarnings("CheckReturnValue") public static class JoinCommandContext extends ParserRuleContext { public Token type; - public TerminalNode DEV_JOIN() { return getToken(EsqlBaseParser.DEV_JOIN, 0); } + public TerminalNode JOIN() { return getToken(EsqlBaseParser.JOIN, 0); } public JoinTargetContext joinTarget() { return getRuleContext(JoinTargetContext.class,0); } public JoinConditionContext joinCondition() { return getRuleContext(JoinConditionContext.class,0); } - public TerminalNode DEV_JOIN_LOOKUP() { return getToken(EsqlBaseParser.DEV_JOIN_LOOKUP, 0); } + public TerminalNode JOIN_LOOKUP() { return getToken(EsqlBaseParser.JOIN_LOOKUP, 0); } public TerminalNode DEV_JOIN_LEFT() { return getToken(EsqlBaseParser.DEV_JOIN_LEFT, 0); } public TerminalNode DEV_JOIN_RIGHT() { return getToken(EsqlBaseParser.DEV_JOIN_RIGHT, 0); } @SuppressWarnings("this-escape") @@ -5615,30 +5634,22 @@ public final JoinCommandContext joinCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(639); - _errHandler.sync(this); + setState(635); + ((JoinCommandContext)_localctx).type = _input.LT(1); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) { - { - setState(638); - ((JoinCommandContext)_localctx).type = _input.LT(1); - _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) ) { - ((JoinCommandContext)_localctx).type = (Token)_errHandler.recoverInline(this); - } - else { - if ( _input.LA(1)==Token.EOF ) matchedEOF = true; - _errHandler.reportMatch(this); - consume(); - } - } + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 12713984L) != 0)) ) { + ((JoinCommandContext)_localctx).type = (Token)_errHandler.recoverInline(this); } - - setState(641); - match(DEV_JOIN); - setState(642); + else { + if ( _input.LA(1)==Token.EOF ) matchedEOF = true; + _errHandler.reportMatch(this); + consume(); + } + setState(636); + match(JOIN); + setState(637); joinTarget(); - setState(643); + setState(638); joinCondition(); } } @@ -5656,14 +5667,9 @@ public final JoinCommandContext joinCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class JoinTargetContext extends ParserRuleContext { public IndexPatternContext index; - public IdentifierContext alias; public IndexPatternContext indexPattern() { return getRuleContext(IndexPatternContext.class,0); } - public TerminalNode AS() { return getToken(EsqlBaseParser.AS, 0); } - public IdentifierContext identifier() { - return getRuleContext(IdentifierContext.class,0); - } @SuppressWarnings("this-escape") public JoinTargetContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -5687,24 +5693,11 @@ public T accept(ParseTreeVisitor visitor) { public final JoinTargetContext joinTarget() throws RecognitionException { JoinTargetContext _localctx = new JoinTargetContext(_ctx, getState()); enterRule(_localctx, 130, RULE_joinTarget); - int _la; try { enterOuterAlt(_localctx, 1); { - setState(645); + setState(640); ((JoinTargetContext)_localctx).index = indexPattern(); - setState(648); - _errHandler.sync(this); - _la = _input.LA(1); - if (_la==AS) { - { - setState(646); - match(AS); - setState(647); - ((JoinTargetContext)_localctx).alias = identifier(); - } - } - } } catch (RecognitionException re) { @@ -5758,27 +5751,27 @@ public final JoinConditionContext joinCondition() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(650); + setState(642); match(ON); - setState(651); + setState(643); joinPredicate(); - setState(656); + setState(648); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,63,_ctx); + _alt = getInterpreter().adaptivePredict(_input,61,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(652); + setState(644); match(COMMA); - setState(653); + setState(645); joinPredicate(); } } } - setState(658); + setState(650); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,63,_ctx); + _alt = getInterpreter().adaptivePredict(_input,61,_ctx); } } } @@ -5824,7 +5817,7 @@ public final JoinPredicateContext joinPredicate() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(659); + setState(651); valueExpression(); } } @@ -5853,10 +5846,6 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); - case 35: - return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex); - case 38: - return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex); } return true; } @@ -5880,53 +5869,37 @@ private boolean processingCommand_sempred(ProcessingCommandContext _localctx, in return this.isDevVersion(); case 3: return this.isDevVersion(); - case 4: - return this.isDevVersion(); } return true; } private boolean booleanExpression_sempred(BooleanExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 5: + case 4: return precpred(_ctx, 5); - case 6: + case 5: return precpred(_ctx, 4); } return true; } private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 7: + case 6: return precpred(_ctx, 2); - case 8: + case 7: return precpred(_ctx, 1); } return true; } private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, int predIndex) { switch (predIndex) { - case 9: + case 8: return precpred(_ctx, 1); } return true; } - private boolean identifierPattern_sempred(IdentifierPatternContext _localctx, int predIndex) { - switch (predIndex) { - case 10: - return this.isDevVersion(); - } - return true; - } - private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _localctx, int predIndex) { - switch (predIndex) { - case 11: - return this.isDevVersion(); - } - return true; - } public static final String _serializedATN = - "\u0004\u0001\u0082\u0296\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0082\u028e\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -5950,397 +5923,390 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca "\u009d\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0003\u0003\u00b1\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004"+ + "\u0003\u0003\u00b0\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0003\u0005\u00bc\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0005\u0005\u00c3\b\u0005\n\u0005\f\u0005\u00c6\t\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00cd"+ + "\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00d2\b\u0005"+ "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0003\u0005\u00bd\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0005\u0005\u00c4\b\u0005\n\u0005\f\u0005\u00c7"+ - "\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+ - "\u0005\u00ce\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00d3"+ - "\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0005\u0005\u00db\b\u0005\n\u0005\f\u0005\u00de\t\u0005\u0001\u0006"+ - "\u0001\u0006\u0003\u0006\u00e2\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0003\u0006\u00e9\b\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0003\u0006\u00ee\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0003\u0007\u00f3\b\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b"+ - "\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00fd\b\b\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0003\t\u0103\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0005\t\u010b\b\t\n\t\f\t\u010e\t\t\u0001\n\u0001\n\u0001\n\u0001\n"+ - "\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0118\b\n\u0001\n\u0001\n\u0001"+ - "\n\u0005\n\u011d\b\n\n\n\f\n\u0120\t\n\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0128\b\u000b\n\u000b"+ - "\f\u000b\u012b\t\u000b\u0001\u000b\u0001\u000b\u0003\u000b\u012f\b\u000b"+ - "\u0003\u000b\u0131\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ - "\r\u0001\r\u0001\r\u0001\r\u0005\r\u013b\b\r\n\r\f\r\u013e\t\r\u0001\r"+ - "\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001"+ - "\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0005\u0011\u014e\b\u0011\n\u0011\f\u0011\u0151\t\u0011\u0001\u0012"+ - "\u0001\u0012\u0001\u0012\u0003\u0012\u0156\b\u0012\u0001\u0012\u0001\u0012"+ - "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u015e\b\u0013"+ - "\n\u0013\f\u0013\u0161\t\u0013\u0001\u0013\u0003\u0013\u0164\b\u0013\u0001"+ - "\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u0169\b\u0014\u0001\u0014\u0001"+ + "\u0005\u0005\u00da\b\u0005\n\u0005\f\u0005\u00dd\t\u0005\u0001\u0006\u0001"+ + "\u0006\u0003\u0006\u00e1\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0006\u0003\u0006\u00e8\b\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0003\u0006\u00ed\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003"+ + "\u0007\u00f2\b\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001"+ + "\b\u0001\b\u0001\b\u0001\b\u0003\b\u00fc\b\b\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0003\t\u0102\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005"+ + "\t\u010a\b\t\n\t\f\t\u010d\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n"+ + "\u0001\n\u0001\n\u0001\n\u0003\n\u0117\b\n\u0001\n\u0001\n\u0001\n\u0005"+ + "\n\u011c\b\n\n\n\f\n\u011f\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ + "\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0127\b\u000b\n\u000b\f\u000b"+ + "\u012a\t\u000b\u0001\u000b\u0001\u000b\u0003\u000b\u012e\b\u000b\u0003"+ + "\u000b\u0130\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001"+ + "\r\u0001\r\u0001\r\u0005\r\u013a\b\r\n\r\f\r\u013d\t\r\u0001\r\u0001\r"+ + "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"+ + "\u0005\u0011\u014d\b\u0011\n\u0011\f\u0011\u0150\t\u0011\u0001\u0012\u0001"+ + "\u0012\u0001\u0012\u0003\u0012\u0155\b\u0012\u0001\u0012\u0001\u0012\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u015d\b\u0013\n"+ + "\u0013\f\u0013\u0160\t\u0013\u0001\u0013\u0003\u0013\u0163\b\u0013\u0001"+ + "\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u0168\b\u0014\u0001\u0014\u0001"+ "\u0014\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0003\u0017\u0173\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"+ - "\u0018\u0005\u0018\u0179\b\u0018\n\u0018\f\u0018\u017c\t\u0018\u0001\u0019"+ + "\u0017\u0003\u0017\u0172\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"+ + "\u0018\u0005\u0018\u0178\b\u0018\n\u0018\f\u0018\u017b\t\u0018\u0001\u0019"+ "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a"+ - "\u0001\u001a\u0005\u001a\u0186\b\u001a\n\u001a\f\u001a\u0189\t\u001a\u0001"+ - "\u001a\u0003\u001a\u018c\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a\u0190"+ + "\u0001\u001a\u0005\u001a\u0185\b\u001a\n\u001a\f\u001a\u0188\t\u001a\u0001"+ + "\u001a\u0003\u001a\u018b\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a\u018f"+ "\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0003"+ - "\u001c\u0197\b\u001c\u0001\u001c\u0001\u001c\u0003\u001c\u019b\b\u001c"+ - "\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u01a0\b\u001d\n\u001d"+ - "\f\u001d\u01a3\t\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0003\u001e"+ - "\u01a8\b\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01ad\b"+ - "\u001f\n\u001f\f\u001f\u01b0\t\u001f\u0001 \u0001 \u0001 \u0005 \u01b5"+ - "\b \n \f \u01b8\t \u0001!\u0001!\u0001!\u0005!\u01bd\b!\n!\f!\u01c0\t"+ - "!\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0003#\u01c7\b#\u0001$\u0001$\u0001"+ - "$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001"+ - "$\u0005$\u01d6\b$\n$\f$\u01d9\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001"+ - "$\u0005$\u01e1\b$\n$\f$\u01e4\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001"+ - "$\u0005$\u01ec\b$\n$\f$\u01ef\t$\u0001$\u0001$\u0003$\u01f3\b$\u0001%"+ - "\u0001%\u0003%\u01f7\b%\u0001&\u0001&\u0001&\u0003&\u01fc\b&\u0001\'\u0001"+ - "\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0005(\u0205\b(\n(\f(\u0208\t("+ - "\u0001)\u0001)\u0003)\u020c\b)\u0001)\u0001)\u0003)\u0210\b)\u0001*\u0001"+ - "*\u0001*\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0005,\u021c"+ - "\b,\n,\f,\u021f\t,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001"+ - ".\u0003.\u0229\b.\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u00010\u0001"+ - "1\u00011\u00011\u00051\u0235\b1\n1\f1\u0238\t1\u00012\u00012\u00012\u0001"+ - "2\u00013\u00013\u00014\u00014\u00034\u0242\b4\u00015\u00035\u0245\b5\u0001"+ - "5\u00015\u00016\u00036\u024a\b6\u00016\u00016\u00017\u00017\u00018\u0001"+ - "8\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001"+ - ";\u0001<\u0001<\u0001<\u0001<\u0003<\u0260\b<\u0001<\u0001<\u0001<\u0001"+ - "<\u0005<\u0266\b<\n<\f<\u0269\t<\u0003<\u026b\b<\u0001=\u0001=\u0001="+ - "\u0003=\u0270\b=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001>\u0001>\u0001"+ - "?\u0001?\u0001?\u0001?\u0003?\u027d\b?\u0001@\u0003@\u0280\b@\u0001@\u0001"+ - "@\u0001@\u0001@\u0001A\u0001A\u0001A\u0003A\u0289\bA\u0001B\u0001B\u0001"+ - "B\u0001B\u0005B\u028f\bB\nB\fB\u0292\tB\u0001C\u0001C\u0001C\u0000\u0004"+ - "\u0002\n\u0012\u0014D\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+ - "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+ - "^`bdfhjlnprtvxz|~\u0080\u0082\u0084\u0086\u0000\t\u0001\u0000@A\u0001"+ - "\u0000BD\u0002\u0000\u001e\u001eSS\u0001\u0000JK\u0002\u0000##((\u0002"+ - "\u0000++..\u0002\u0000**88\u0002\u000099;?\u0001\u0000\u0016\u0018\u02b0"+ - "\u0000\u0088\u0001\u0000\u0000\u0000\u0002\u008b\u0001\u0000\u0000\u0000"+ - "\u0004\u009c\u0001\u0000\u0000\u0000\u0006\u00b0\u0001\u0000\u0000\u0000"+ - "\b\u00b2\u0001\u0000\u0000\u0000\n\u00d2\u0001\u0000\u0000\u0000\f\u00ed"+ - "\u0001\u0000\u0000\u0000\u000e\u00ef\u0001\u0000\u0000\u0000\u0010\u00fc"+ - "\u0001\u0000\u0000\u0000\u0012\u0102\u0001\u0000\u0000\u0000\u0014\u0117"+ - "\u0001\u0000\u0000\u0000\u0016\u0121\u0001\u0000\u0000\u0000\u0018\u0134"+ - "\u0001\u0000\u0000\u0000\u001a\u0136\u0001\u0000\u0000\u0000\u001c\u0141"+ - "\u0001\u0000\u0000\u0000\u001e\u0145\u0001\u0000\u0000\u0000 \u0147\u0001"+ - "\u0000\u0000\u0000\"\u014a\u0001\u0000\u0000\u0000$\u0155\u0001\u0000"+ - "\u0000\u0000&\u0159\u0001\u0000\u0000\u0000(\u0168\u0001\u0000\u0000\u0000"+ - "*\u016c\u0001\u0000\u0000\u0000,\u016e\u0001\u0000\u0000\u0000.\u0172"+ - "\u0001\u0000\u0000\u00000\u0174\u0001\u0000\u0000\u00002\u017d\u0001\u0000"+ - "\u0000\u00004\u0181\u0001\u0000\u0000\u00006\u0191\u0001\u0000\u0000\u0000"+ - "8\u0194\u0001\u0000\u0000\u0000:\u019c\u0001\u0000\u0000\u0000<\u01a4"+ - "\u0001\u0000\u0000\u0000>\u01a9\u0001\u0000\u0000\u0000@\u01b1\u0001\u0000"+ - "\u0000\u0000B\u01b9\u0001\u0000\u0000\u0000D\u01c1\u0001\u0000\u0000\u0000"+ - "F\u01c6\u0001\u0000\u0000\u0000H\u01f2\u0001\u0000\u0000\u0000J\u01f6"+ - "\u0001\u0000\u0000\u0000L\u01fb\u0001\u0000\u0000\u0000N\u01fd\u0001\u0000"+ - "\u0000\u0000P\u0200\u0001\u0000\u0000\u0000R\u0209\u0001\u0000\u0000\u0000"+ - "T\u0211\u0001\u0000\u0000\u0000V\u0214\u0001\u0000\u0000\u0000X\u0217"+ - "\u0001\u0000\u0000\u0000Z\u0220\u0001\u0000\u0000\u0000\\\u0224\u0001"+ - "\u0000\u0000\u0000^\u022a\u0001\u0000\u0000\u0000`\u022e\u0001\u0000\u0000"+ - "\u0000b\u0231\u0001\u0000\u0000\u0000d\u0239\u0001\u0000\u0000\u0000f"+ - "\u023d\u0001\u0000\u0000\u0000h\u0241\u0001\u0000\u0000\u0000j\u0244\u0001"+ - "\u0000\u0000\u0000l\u0249\u0001\u0000\u0000\u0000n\u024d\u0001\u0000\u0000"+ - "\u0000p\u024f\u0001\u0000\u0000\u0000r\u0251\u0001\u0000\u0000\u0000t"+ - "\u0254\u0001\u0000\u0000\u0000v\u0258\u0001\u0000\u0000\u0000x\u025b\u0001"+ - "\u0000\u0000\u0000z\u026f\u0001\u0000\u0000\u0000|\u0273\u0001\u0000\u0000"+ - "\u0000~\u0278\u0001\u0000\u0000\u0000\u0080\u027f\u0001\u0000\u0000\u0000"+ - "\u0082\u0285\u0001\u0000\u0000\u0000\u0084\u028a\u0001\u0000\u0000\u0000"+ - "\u0086\u0293\u0001\u0000\u0000\u0000\u0088\u0089\u0003\u0002\u0001\u0000"+ - "\u0089\u008a\u0005\u0000\u0000\u0001\u008a\u0001\u0001\u0000\u0000\u0000"+ - "\u008b\u008c\u0006\u0001\uffff\uffff\u0000\u008c\u008d\u0003\u0004\u0002"+ - "\u0000\u008d\u0093\u0001\u0000\u0000\u0000\u008e\u008f\n\u0001\u0000\u0000"+ - "\u008f\u0090\u0005\u001d\u0000\u0000\u0090\u0092\u0003\u0006\u0003\u0000"+ - "\u0091\u008e\u0001\u0000\u0000\u0000\u0092\u0095\u0001\u0000\u0000\u0000"+ - "\u0093\u0091\u0001\u0000\u0000\u0000\u0093\u0094\u0001\u0000\u0000\u0000"+ - "\u0094\u0003\u0001\u0000\u0000\u0000\u0095\u0093\u0001\u0000\u0000\u0000"+ - "\u0096\u009d\u0003r9\u0000\u0097\u009d\u0003&\u0013\u0000\u0098\u009d"+ - "\u0003 \u0010\u0000\u0099\u009d\u0003v;\u0000\u009a\u009b\u0004\u0002"+ - "\u0001\u0000\u009b\u009d\u00034\u001a\u0000\u009c\u0096\u0001\u0000\u0000"+ - "\u0000\u009c\u0097\u0001\u0000\u0000\u0000\u009c\u0098\u0001\u0000\u0000"+ - "\u0000\u009c\u0099\u0001\u0000\u0000\u0000\u009c\u009a\u0001\u0000\u0000"+ - "\u0000\u009d\u0005\u0001\u0000\u0000\u0000\u009e\u00b1\u00036\u001b\u0000"+ - "\u009f\u00b1\u0003\b\u0004\u0000\u00a0\u00b1\u0003T*\u0000\u00a1\u00b1"+ - "\u0003N\'\u0000\u00a2\u00b1\u00038\u001c\u0000\u00a3\u00b1\u0003P(\u0000"+ - "\u00a4\u00b1\u0003V+\u0000\u00a5\u00b1\u0003X,\u0000\u00a6\u00b1\u0003"+ - "\\.\u0000\u00a7\u00b1\u0003^/\u0000\u00a8\u00b1\u0003x<\u0000\u00a9\u00b1"+ - "\u0003`0\u0000\u00aa\u00ab\u0004\u0003\u0002\u0000\u00ab\u00b1\u0003~"+ - "?\u0000\u00ac\u00ad\u0004\u0003\u0003\u0000\u00ad\u00b1\u0003|>\u0000"+ - "\u00ae\u00af\u0004\u0003\u0004\u0000\u00af\u00b1\u0003\u0080@\u0000\u00b0"+ - "\u009e\u0001\u0000\u0000\u0000\u00b0\u009f\u0001\u0000\u0000\u0000\u00b0"+ - "\u00a0\u0001\u0000\u0000\u0000\u00b0\u00a1\u0001\u0000\u0000\u0000\u00b0"+ - "\u00a2\u0001\u0000\u0000\u0000\u00b0\u00a3\u0001\u0000\u0000\u0000\u00b0"+ - "\u00a4\u0001\u0000\u0000\u0000\u00b0\u00a5\u0001\u0000\u0000\u0000\u00b0"+ - "\u00a6\u0001\u0000\u0000\u0000\u00b0\u00a7\u0001\u0000\u0000\u0000\u00b0"+ - "\u00a8\u0001\u0000\u0000\u0000\u00b0\u00a9\u0001\u0000\u0000\u0000\u00b0"+ - "\u00aa\u0001\u0000\u0000\u0000\u00b0\u00ac\u0001\u0000\u0000\u0000\u00b0"+ - "\u00ae\u0001\u0000\u0000\u0000\u00b1\u0007\u0001\u0000\u0000\u0000\u00b2"+ - "\u00b3\u0005\u0010\u0000\u0000\u00b3\u00b4\u0003\n\u0005\u0000\u00b4\t"+ - "\u0001\u0000\u0000\u0000\u00b5\u00b6\u0006\u0005\uffff\uffff\u0000\u00b6"+ - "\u00b7\u00051\u0000\u0000\u00b7\u00d3\u0003\n\u0005\b\u00b8\u00d3\u0003"+ - "\u0010\b\u0000\u00b9\u00d3\u0003\f\u0006\u0000\u00ba\u00bc\u0003\u0010"+ - "\b\u0000\u00bb\u00bd\u00051\u0000\u0000\u00bc\u00bb\u0001\u0000\u0000"+ - "\u0000\u00bc\u00bd\u0001\u0000\u0000\u0000\u00bd\u00be\u0001\u0000\u0000"+ - "\u0000\u00be\u00bf\u0005,\u0000\u0000\u00bf\u00c0\u00050\u0000\u0000\u00c0"+ - "\u00c5\u0003\u0010\b\u0000\u00c1\u00c2\u0005\'\u0000\u0000\u00c2\u00c4"+ - "\u0003\u0010\b\u0000\u00c3\u00c1\u0001\u0000\u0000\u0000\u00c4\u00c7\u0001"+ - "\u0000\u0000\u0000\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001"+ - "\u0000\u0000\u0000\u00c6\u00c8\u0001\u0000\u0000\u0000\u00c7\u00c5\u0001"+ - "\u0000\u0000\u0000\u00c8\u00c9\u00057\u0000\u0000\u00c9\u00d3\u0001\u0000"+ - "\u0000\u0000\u00ca\u00cb\u0003\u0010\b\u0000\u00cb\u00cd\u0005-\u0000"+ - "\u0000\u00cc\u00ce\u00051\u0000\u0000\u00cd\u00cc\u0001\u0000\u0000\u0000"+ - "\u00cd\u00ce\u0001\u0000\u0000\u0000\u00ce\u00cf\u0001\u0000\u0000\u0000"+ - "\u00cf\u00d0\u00052\u0000\u0000\u00d0\u00d3\u0001\u0000\u0000\u0000\u00d1"+ - "\u00d3\u0003\u000e\u0007\u0000\u00d2\u00b5\u0001\u0000\u0000\u0000\u00d2"+ - "\u00b8\u0001\u0000\u0000\u0000\u00d2\u00b9\u0001\u0000\u0000\u0000\u00d2"+ - "\u00ba\u0001\u0000\u0000\u0000\u00d2\u00ca\u0001\u0000\u0000\u0000\u00d2"+ - "\u00d1\u0001\u0000\u0000\u0000\u00d3\u00dc\u0001\u0000\u0000\u0000\u00d4"+ - "\u00d5\n\u0005\u0000\u0000\u00d5\u00d6\u0005\"\u0000\u0000\u00d6\u00db"+ - "\u0003\n\u0005\u0006\u00d7\u00d8\n\u0004\u0000\u0000\u00d8\u00d9\u0005"+ - "4\u0000\u0000\u00d9\u00db\u0003\n\u0005\u0005\u00da\u00d4\u0001\u0000"+ - "\u0000\u0000\u00da\u00d7\u0001\u0000\u0000\u0000\u00db\u00de\u0001\u0000"+ - "\u0000\u0000\u00dc\u00da\u0001\u0000\u0000\u0000\u00dc\u00dd\u0001\u0000"+ - "\u0000\u0000\u00dd\u000b\u0001\u0000\u0000\u0000\u00de\u00dc\u0001\u0000"+ - "\u0000\u0000\u00df\u00e1\u0003\u0010\b\u0000\u00e0\u00e2\u00051\u0000"+ - "\u0000\u00e1\u00e0\u0001\u0000\u0000\u0000\u00e1\u00e2\u0001\u0000\u0000"+ - "\u0000\u00e2\u00e3\u0001\u0000\u0000\u0000\u00e3\u00e4\u0005/\u0000\u0000"+ - "\u00e4\u00e5\u0003n7\u0000\u00e5\u00ee\u0001\u0000\u0000\u0000\u00e6\u00e8"+ - "\u0003\u0010\b\u0000\u00e7\u00e9\u00051\u0000\u0000\u00e8\u00e7\u0001"+ - "\u0000\u0000\u0000\u00e8\u00e9\u0001\u0000\u0000\u0000\u00e9\u00ea\u0001"+ - "\u0000\u0000\u0000\u00ea\u00eb\u00056\u0000\u0000\u00eb\u00ec\u0003n7"+ - "\u0000\u00ec\u00ee\u0001\u0000\u0000\u0000\u00ed\u00df\u0001\u0000\u0000"+ - "\u0000\u00ed\u00e6\u0001\u0000\u0000\u0000\u00ee\r\u0001\u0000\u0000\u0000"+ - "\u00ef\u00f2\u0003>\u001f\u0000\u00f0\u00f1\u0005%\u0000\u0000\u00f1\u00f3"+ - "\u0003\u001e\u000f\u0000\u00f2\u00f0\u0001\u0000\u0000\u0000\u00f2\u00f3"+ - "\u0001\u0000\u0000\u0000\u00f3\u00f4\u0001\u0000\u0000\u0000\u00f4\u00f5"+ - "\u0005&\u0000\u0000\u00f5\u00f6\u0003H$\u0000\u00f6\u000f\u0001\u0000"+ - "\u0000\u0000\u00f7\u00fd\u0003\u0012\t\u0000\u00f8\u00f9\u0003\u0012\t"+ - "\u0000\u00f9\u00fa\u0003p8\u0000\u00fa\u00fb\u0003\u0012\t\u0000\u00fb"+ - "\u00fd\u0001\u0000\u0000\u0000\u00fc\u00f7\u0001\u0000\u0000\u0000\u00fc"+ - "\u00f8\u0001\u0000\u0000\u0000\u00fd\u0011\u0001\u0000\u0000\u0000\u00fe"+ - "\u00ff\u0006\t\uffff\uffff\u0000\u00ff\u0103\u0003\u0014\n\u0000\u0100"+ - "\u0101\u0007\u0000\u0000\u0000\u0101\u0103\u0003\u0012\t\u0003\u0102\u00fe"+ - "\u0001\u0000\u0000\u0000\u0102\u0100\u0001\u0000\u0000\u0000\u0103\u010c"+ - "\u0001\u0000\u0000\u0000\u0104\u0105\n\u0002\u0000\u0000\u0105\u0106\u0007"+ - "\u0001\u0000\u0000\u0106\u010b\u0003\u0012\t\u0003\u0107\u0108\n\u0001"+ - "\u0000\u0000\u0108\u0109\u0007\u0000\u0000\u0000\u0109\u010b\u0003\u0012"+ - "\t\u0002\u010a\u0104\u0001\u0000\u0000\u0000\u010a\u0107\u0001\u0000\u0000"+ - "\u0000\u010b\u010e\u0001\u0000\u0000\u0000\u010c\u010a\u0001\u0000\u0000"+ - "\u0000\u010c\u010d\u0001\u0000\u0000\u0000\u010d\u0013\u0001\u0000\u0000"+ - "\u0000\u010e\u010c\u0001\u0000\u0000\u0000\u010f\u0110\u0006\n\uffff\uffff"+ - "\u0000\u0110\u0118\u0003H$\u0000\u0111\u0118\u0003>\u001f\u0000\u0112"+ - "\u0118\u0003\u0016\u000b\u0000\u0113\u0114\u00050\u0000\u0000\u0114\u0115"+ - "\u0003\n\u0005\u0000\u0115\u0116\u00057\u0000\u0000\u0116\u0118\u0001"+ - "\u0000\u0000\u0000\u0117\u010f\u0001\u0000\u0000\u0000\u0117\u0111\u0001"+ - "\u0000\u0000\u0000\u0117\u0112\u0001\u0000\u0000\u0000\u0117\u0113\u0001"+ - "\u0000\u0000\u0000\u0118\u011e\u0001\u0000\u0000\u0000\u0119\u011a\n\u0001"+ - "\u0000\u0000\u011a\u011b\u0005%\u0000\u0000\u011b\u011d\u0003\u001e\u000f"+ - "\u0000\u011c\u0119\u0001\u0000\u0000\u0000\u011d\u0120\u0001\u0000\u0000"+ - "\u0000\u011e\u011c\u0001\u0000\u0000\u0000\u011e\u011f\u0001\u0000\u0000"+ - "\u0000\u011f\u0015\u0001\u0000\u0000\u0000\u0120\u011e\u0001\u0000\u0000"+ - "\u0000\u0121\u0122\u0003\u0018\f\u0000\u0122\u0130\u00050\u0000\u0000"+ - "\u0123\u0131\u0005B\u0000\u0000\u0124\u0129\u0003\n\u0005\u0000\u0125"+ - "\u0126\u0005\'\u0000\u0000\u0126\u0128\u0003\n\u0005\u0000\u0127\u0125"+ - "\u0001\u0000\u0000\u0000\u0128\u012b\u0001\u0000\u0000\u0000\u0129\u0127"+ - "\u0001\u0000\u0000\u0000\u0129\u012a\u0001\u0000\u0000\u0000\u012a\u012e"+ - "\u0001\u0000\u0000\u0000\u012b\u0129\u0001\u0000\u0000\u0000\u012c\u012d"+ - "\u0005\'\u0000\u0000\u012d\u012f\u0003\u001a\r\u0000\u012e\u012c\u0001"+ - "\u0000\u0000\u0000\u012e\u012f\u0001\u0000\u0000\u0000\u012f\u0131\u0001"+ - "\u0000\u0000\u0000\u0130\u0123\u0001\u0000\u0000\u0000\u0130\u0124\u0001"+ - "\u0000\u0000\u0000\u0130\u0131\u0001\u0000\u0000\u0000\u0131\u0132\u0001"+ - "\u0000\u0000\u0000\u0132\u0133\u00057\u0000\u0000\u0133\u0017\u0001\u0000"+ - "\u0000\u0000\u0134\u0135\u0003L&\u0000\u0135\u0019\u0001\u0000\u0000\u0000"+ - "\u0136\u0137\u0005E\u0000\u0000\u0137\u013c\u0003\u001c\u000e\u0000\u0138"+ - "\u0139\u0005\'\u0000\u0000\u0139\u013b\u0003\u001c\u000e\u0000\u013a\u0138"+ - "\u0001\u0000\u0000\u0000\u013b\u013e\u0001\u0000\u0000\u0000\u013c\u013a"+ - "\u0001\u0000\u0000\u0000\u013c\u013d\u0001\u0000\u0000\u0000\u013d\u013f"+ - "\u0001\u0000\u0000\u0000\u013e\u013c\u0001\u0000\u0000\u0000\u013f\u0140"+ - "\u0005F\u0000\u0000\u0140\u001b\u0001\u0000\u0000\u0000\u0141\u0142\u0003"+ - "n7\u0000\u0142\u0143\u0005&\u0000\u0000\u0143\u0144\u0003H$\u0000\u0144"+ - "\u001d\u0001\u0000\u0000\u0000\u0145\u0146\u0003D\"\u0000\u0146\u001f"+ - "\u0001\u0000\u0000\u0000\u0147\u0148\u0005\f\u0000\u0000\u0148\u0149\u0003"+ - "\"\u0011\u0000\u0149!\u0001\u0000\u0000\u0000\u014a\u014f\u0003$\u0012"+ - "\u0000\u014b\u014c\u0005\'\u0000\u0000\u014c\u014e\u0003$\u0012\u0000"+ - "\u014d\u014b\u0001\u0000\u0000\u0000\u014e\u0151\u0001\u0000\u0000\u0000"+ - "\u014f\u014d\u0001\u0000\u0000\u0000\u014f\u0150\u0001\u0000\u0000\u0000"+ - "\u0150#\u0001\u0000\u0000\u0000\u0151\u014f\u0001\u0000\u0000\u0000\u0152"+ - "\u0153\u0003>\u001f\u0000\u0153\u0154\u0005$\u0000\u0000\u0154\u0156\u0001"+ - "\u0000\u0000\u0000\u0155\u0152\u0001\u0000\u0000\u0000\u0155\u0156\u0001"+ - "\u0000\u0000\u0000\u0156\u0157\u0001\u0000\u0000\u0000\u0157\u0158\u0003"+ - "\n\u0005\u0000\u0158%\u0001\u0000\u0000\u0000\u0159\u015a\u0005\u0006"+ - "\u0000\u0000\u015a\u015f\u0003(\u0014\u0000\u015b\u015c\u0005\'\u0000"+ - "\u0000\u015c\u015e\u0003(\u0014\u0000\u015d\u015b\u0001\u0000\u0000\u0000"+ - "\u015e\u0161\u0001\u0000\u0000\u0000\u015f\u015d\u0001\u0000\u0000\u0000"+ - "\u015f\u0160\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000\u0000"+ - "\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0164\u0003.\u0017\u0000\u0163"+ - "\u0162\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000\u0000\u0164"+ - "\'\u0001\u0000\u0000\u0000\u0165\u0166\u0003*\u0015\u0000\u0166\u0167"+ - "\u0005&\u0000\u0000\u0167\u0169\u0001\u0000\u0000\u0000\u0168\u0165\u0001"+ - "\u0000\u0000\u0000\u0168\u0169\u0001\u0000\u0000\u0000\u0169\u016a\u0001"+ - "\u0000\u0000\u0000\u016a\u016b\u0003,\u0016\u0000\u016b)\u0001\u0000\u0000"+ - "\u0000\u016c\u016d\u0005S\u0000\u0000\u016d+\u0001\u0000\u0000\u0000\u016e"+ - "\u016f\u0007\u0002\u0000\u0000\u016f-\u0001\u0000\u0000\u0000\u0170\u0173"+ - "\u00030\u0018\u0000\u0171\u0173\u00032\u0019\u0000\u0172\u0170\u0001\u0000"+ - "\u0000\u0000\u0172\u0171\u0001\u0000\u0000\u0000\u0173/\u0001\u0000\u0000"+ - "\u0000\u0174\u0175\u0005R\u0000\u0000\u0175\u017a\u0005S\u0000\u0000\u0176"+ - "\u0177\u0005\'\u0000\u0000\u0177\u0179\u0005S\u0000\u0000\u0178\u0176"+ - "\u0001\u0000\u0000\u0000\u0179\u017c\u0001\u0000\u0000\u0000\u017a\u0178"+ - "\u0001\u0000\u0000\u0000\u017a\u017b\u0001\u0000\u0000\u0000\u017b1\u0001"+ - "\u0000\u0000\u0000\u017c\u017a\u0001\u0000\u0000\u0000\u017d\u017e\u0005"+ - "H\u0000\u0000\u017e\u017f\u00030\u0018\u0000\u017f\u0180\u0005I\u0000"+ - "\u0000\u01803\u0001\u0000\u0000\u0000\u0181\u0182\u0005\u0013\u0000\u0000"+ - "\u0182\u0187\u0003(\u0014\u0000\u0183\u0184\u0005\'\u0000\u0000\u0184"+ - "\u0186\u0003(\u0014\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0186\u0189"+ - "\u0001\u0000\u0000\u0000\u0187\u0185\u0001\u0000\u0000\u0000\u0187\u0188"+ - "\u0001\u0000\u0000\u0000\u0188\u018b\u0001\u0000\u0000\u0000\u0189\u0187"+ - "\u0001\u0000\u0000\u0000\u018a\u018c\u0003:\u001d\u0000\u018b\u018a\u0001"+ - "\u0000\u0000\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018f\u0001"+ - "\u0000\u0000\u0000\u018d\u018e\u0005!\u0000\u0000\u018e\u0190\u0003\""+ - "\u0011\u0000\u018f\u018d\u0001\u0000\u0000\u0000\u018f\u0190\u0001\u0000"+ - "\u0000\u0000\u01905\u0001\u0000\u0000\u0000\u0191\u0192\u0005\u0004\u0000"+ - "\u0000\u0192\u0193\u0003\"\u0011\u0000\u01937\u0001\u0000\u0000\u0000"+ - "\u0194\u0196\u0005\u000f\u0000\u0000\u0195\u0197\u0003:\u001d\u0000\u0196"+ - "\u0195\u0001\u0000\u0000\u0000\u0196\u0197\u0001\u0000\u0000\u0000\u0197"+ - "\u019a\u0001\u0000\u0000\u0000\u0198\u0199\u0005!\u0000\u0000\u0199\u019b"+ - "\u0003\"\u0011\u0000\u019a\u0198\u0001\u0000\u0000\u0000\u019a\u019b\u0001"+ - "\u0000\u0000\u0000\u019b9\u0001\u0000\u0000\u0000\u019c\u01a1\u0003<\u001e"+ - "\u0000\u019d\u019e\u0005\'\u0000\u0000\u019e\u01a0\u0003<\u001e\u0000"+ - "\u019f\u019d\u0001\u0000\u0000\u0000\u01a0\u01a3\u0001\u0000\u0000\u0000"+ - "\u01a1\u019f\u0001\u0000\u0000\u0000\u01a1\u01a2\u0001\u0000\u0000\u0000"+ - "\u01a2;\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000\u0000\u01a4"+ - "\u01a7\u0003$\u0012\u0000\u01a5\u01a6\u0005\u0010\u0000\u0000\u01a6\u01a8"+ - "\u0003\n\u0005\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7\u01a8\u0001"+ - "\u0000\u0000\u0000\u01a8=\u0001\u0000\u0000\u0000\u01a9\u01ae\u0003L&"+ - "\u0000\u01aa\u01ab\u0005)\u0000\u0000\u01ab\u01ad\u0003L&\u0000\u01ac"+ - "\u01aa\u0001\u0000\u0000\u0000\u01ad\u01b0\u0001\u0000\u0000\u0000\u01ae"+ - "\u01ac\u0001\u0000\u0000\u0000\u01ae\u01af\u0001\u0000\u0000\u0000\u01af"+ - "?\u0001\u0000\u0000\u0000\u01b0\u01ae\u0001\u0000\u0000\u0000\u01b1\u01b6"+ - "\u0003F#\u0000\u01b2\u01b3\u0005)\u0000\u0000\u01b3\u01b5\u0003F#\u0000"+ - "\u01b4\u01b2\u0001\u0000\u0000\u0000\u01b5\u01b8\u0001\u0000\u0000\u0000"+ - "\u01b6\u01b4\u0001\u0000\u0000\u0000\u01b6\u01b7\u0001\u0000\u0000\u0000"+ - "\u01b7A\u0001\u0000\u0000\u0000\u01b8\u01b6\u0001\u0000\u0000\u0000\u01b9"+ - "\u01be\u0003@ \u0000\u01ba\u01bb\u0005\'\u0000\u0000\u01bb\u01bd\u0003"+ - "@ \u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bd\u01c0\u0001\u0000\u0000"+ - "\u0000\u01be\u01bc\u0001\u0000\u0000\u0000\u01be\u01bf\u0001\u0000\u0000"+ - "\u0000\u01bfC\u0001\u0000\u0000\u0000\u01c0\u01be\u0001\u0000\u0000\u0000"+ - "\u01c1\u01c2\u0007\u0003\u0000\u0000\u01c2E\u0001\u0000\u0000\u0000\u01c3"+ - "\u01c7\u0005W\u0000\u0000\u01c4\u01c5\u0004#\n\u0000\u01c5\u01c7\u0003"+ - "J%\u0000\u01c6\u01c3\u0001\u0000\u0000\u0000\u01c6\u01c4\u0001\u0000\u0000"+ - "\u0000\u01c7G\u0001\u0000\u0000\u0000\u01c8\u01f3\u00052\u0000\u0000\u01c9"+ - "\u01ca\u0003l6\u0000\u01ca\u01cb\u0005J\u0000\u0000\u01cb\u01f3\u0001"+ - "\u0000\u0000\u0000\u01cc\u01f3\u0003j5\u0000\u01cd\u01f3\u0003l6\u0000"+ - "\u01ce\u01f3\u0003f3\u0000\u01cf\u01f3\u0003J%\u0000\u01d0\u01f3\u0003"+ - "n7\u0000\u01d1\u01d2\u0005H\u0000\u0000\u01d2\u01d7\u0003h4\u0000\u01d3"+ - "\u01d4\u0005\'\u0000\u0000\u01d4\u01d6\u0003h4\u0000\u01d5\u01d3\u0001"+ - "\u0000\u0000\u0000\u01d6\u01d9\u0001\u0000\u0000\u0000\u01d7\u01d5\u0001"+ - "\u0000\u0000\u0000\u01d7\u01d8\u0001\u0000\u0000\u0000\u01d8\u01da\u0001"+ - "\u0000\u0000\u0000\u01d9\u01d7\u0001\u0000\u0000\u0000\u01da\u01db\u0005"+ - "I\u0000\u0000\u01db\u01f3\u0001\u0000\u0000\u0000\u01dc\u01dd\u0005H\u0000"+ - "\u0000\u01dd\u01e2\u0003f3\u0000\u01de\u01df\u0005\'\u0000\u0000\u01df"+ - "\u01e1\u0003f3\u0000\u01e0\u01de\u0001\u0000\u0000\u0000\u01e1\u01e4\u0001"+ - "\u0000\u0000\u0000\u01e2\u01e0\u0001\u0000\u0000\u0000\u01e2\u01e3\u0001"+ - "\u0000\u0000\u0000\u01e3\u01e5\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001"+ - "\u0000\u0000\u0000\u01e5\u01e6\u0005I\u0000\u0000\u01e6\u01f3\u0001\u0000"+ - "\u0000\u0000\u01e7\u01e8\u0005H\u0000\u0000\u01e8\u01ed\u0003n7\u0000"+ - "\u01e9\u01ea\u0005\'\u0000\u0000\u01ea\u01ec\u0003n7\u0000\u01eb\u01e9"+ - "\u0001\u0000\u0000\u0000\u01ec\u01ef\u0001\u0000\u0000\u0000\u01ed\u01eb"+ - "\u0001\u0000\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000\u0000\u01ee\u01f0"+ - "\u0001\u0000\u0000\u0000\u01ef\u01ed\u0001\u0000\u0000\u0000\u01f0\u01f1"+ - "\u0005I\u0000\u0000\u01f1\u01f3\u0001\u0000\u0000\u0000\u01f2\u01c8\u0001"+ - "\u0000\u0000\u0000\u01f2\u01c9\u0001\u0000\u0000\u0000\u01f2\u01cc\u0001"+ - "\u0000\u0000\u0000\u01f2\u01cd\u0001\u0000\u0000\u0000\u01f2\u01ce\u0001"+ - "\u0000\u0000\u0000\u01f2\u01cf\u0001\u0000\u0000\u0000\u01f2\u01d0\u0001"+ - "\u0000\u0000\u0000\u01f2\u01d1\u0001\u0000\u0000\u0000\u01f2\u01dc\u0001"+ - "\u0000\u0000\u0000\u01f2\u01e7\u0001\u0000\u0000\u0000\u01f3I\u0001\u0000"+ - "\u0000\u0000\u01f4\u01f7\u00055\u0000\u0000\u01f5\u01f7\u0005G\u0000\u0000"+ - "\u01f6\u01f4\u0001\u0000\u0000\u0000\u01f6\u01f5\u0001\u0000\u0000\u0000"+ - "\u01f7K\u0001\u0000\u0000\u0000\u01f8\u01fc\u0003D\"\u0000\u01f9\u01fa"+ - "\u0004&\u000b\u0000\u01fa\u01fc\u0003J%\u0000\u01fb\u01f8\u0001\u0000"+ - "\u0000\u0000\u01fb\u01f9\u0001\u0000\u0000\u0000\u01fcM\u0001\u0000\u0000"+ - "\u0000\u01fd\u01fe\u0005\t\u0000\u0000\u01fe\u01ff\u0005\u001f\u0000\u0000"+ - "\u01ffO\u0001\u0000\u0000\u0000\u0200\u0201\u0005\u000e\u0000\u0000\u0201"+ - "\u0206\u0003R)\u0000\u0202\u0203\u0005\'\u0000\u0000\u0203\u0205\u0003"+ - "R)\u0000\u0204\u0202\u0001\u0000\u0000\u0000\u0205\u0208\u0001\u0000\u0000"+ - "\u0000\u0206\u0204\u0001\u0000\u0000\u0000\u0206\u0207\u0001\u0000\u0000"+ - "\u0000\u0207Q\u0001\u0000\u0000\u0000\u0208\u0206\u0001\u0000\u0000\u0000"+ - "\u0209\u020b\u0003\n\u0005\u0000\u020a\u020c\u0007\u0004\u0000\u0000\u020b"+ - "\u020a\u0001\u0000\u0000\u0000\u020b\u020c\u0001\u0000\u0000\u0000\u020c"+ - "\u020f\u0001\u0000\u0000\u0000\u020d\u020e\u00053\u0000\u0000\u020e\u0210"+ - "\u0007\u0005\u0000\u0000\u020f\u020d\u0001\u0000\u0000\u0000\u020f\u0210"+ - "\u0001\u0000\u0000\u0000\u0210S\u0001\u0000\u0000\u0000\u0211\u0212\u0005"+ - "\b\u0000\u0000\u0212\u0213\u0003B!\u0000\u0213U\u0001\u0000\u0000\u0000"+ - "\u0214\u0215\u0005\u0002\u0000\u0000\u0215\u0216\u0003B!\u0000\u0216W"+ - "\u0001\u0000\u0000\u0000\u0217\u0218\u0005\u000b\u0000\u0000\u0218\u021d"+ - "\u0003Z-\u0000\u0219\u021a\u0005\'\u0000\u0000\u021a\u021c\u0003Z-\u0000"+ - "\u021b\u0219\u0001\u0000\u0000\u0000\u021c\u021f\u0001\u0000\u0000\u0000"+ - "\u021d\u021b\u0001\u0000\u0000\u0000\u021d\u021e\u0001\u0000\u0000\u0000"+ - "\u021eY\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000\u0000\u0220"+ - "\u0221\u0003@ \u0000\u0221\u0222\u0005[\u0000\u0000\u0222\u0223\u0003"+ - "@ \u0000\u0223[\u0001\u0000\u0000\u0000\u0224\u0225\u0005\u0001\u0000"+ - "\u0000\u0225\u0226\u0003\u0014\n\u0000\u0226\u0228\u0003n7\u0000\u0227"+ - "\u0229\u0003b1\u0000\u0228\u0227\u0001\u0000\u0000\u0000\u0228\u0229\u0001"+ - "\u0000\u0000\u0000\u0229]\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u0007"+ - "\u0000\u0000\u022b\u022c\u0003\u0014\n\u0000\u022c\u022d\u0003n7\u0000"+ - "\u022d_\u0001\u0000\u0000\u0000\u022e\u022f\u0005\n\u0000\u0000\u022f"+ - "\u0230\u0003>\u001f\u0000\u0230a\u0001\u0000\u0000\u0000\u0231\u0236\u0003"+ - "d2\u0000\u0232\u0233\u0005\'\u0000\u0000\u0233\u0235\u0003d2\u0000\u0234"+ - "\u0232\u0001\u0000\u0000\u0000\u0235\u0238\u0001\u0000\u0000\u0000\u0236"+ - "\u0234\u0001\u0000\u0000\u0000\u0236\u0237\u0001\u0000\u0000\u0000\u0237"+ - "c\u0001\u0000\u0000\u0000\u0238\u0236\u0001\u0000\u0000\u0000\u0239\u023a"+ - "\u0003D\"\u0000\u023a\u023b\u0005$\u0000\u0000\u023b\u023c\u0003H$\u0000"+ - "\u023ce\u0001\u0000\u0000\u0000\u023d\u023e\u0007\u0006\u0000\u0000\u023e"+ - "g\u0001\u0000\u0000\u0000\u023f\u0242\u0003j5\u0000\u0240\u0242\u0003"+ - "l6\u0000\u0241\u023f\u0001\u0000\u0000\u0000\u0241\u0240\u0001\u0000\u0000"+ - "\u0000\u0242i\u0001\u0000\u0000\u0000\u0243\u0245\u0007\u0000\u0000\u0000"+ - "\u0244\u0243\u0001\u0000\u0000\u0000\u0244\u0245\u0001\u0000\u0000\u0000"+ - "\u0245\u0246\u0001\u0000\u0000\u0000\u0246\u0247\u0005 \u0000\u0000\u0247"+ - "k\u0001\u0000\u0000\u0000\u0248\u024a\u0007\u0000\u0000\u0000\u0249\u0248"+ - "\u0001\u0000\u0000\u0000\u0249\u024a\u0001\u0000\u0000\u0000\u024a\u024b"+ - "\u0001\u0000\u0000\u0000\u024b\u024c\u0005\u001f\u0000\u0000\u024cm\u0001"+ - "\u0000\u0000\u0000\u024d\u024e\u0005\u001e\u0000\u0000\u024eo\u0001\u0000"+ - "\u0000\u0000\u024f\u0250\u0007\u0007\u0000\u0000\u0250q\u0001\u0000\u0000"+ - "\u0000\u0251\u0252\u0005\u0005\u0000\u0000\u0252\u0253\u0003t:\u0000\u0253"+ - "s\u0001\u0000\u0000\u0000\u0254\u0255\u0005H\u0000\u0000\u0255\u0256\u0003"+ - "\u0002\u0001\u0000\u0256\u0257\u0005I\u0000\u0000\u0257u\u0001\u0000\u0000"+ - "\u0000\u0258\u0259\u0005\r\u0000\u0000\u0259\u025a\u0005k\u0000\u0000"+ - "\u025aw\u0001\u0000\u0000\u0000\u025b\u025c\u0005\u0003\u0000\u0000\u025c"+ - "\u025f\u0005a\u0000\u0000\u025d\u025e\u0005_\u0000\u0000\u025e\u0260\u0003"+ - "@ \u0000\u025f\u025d\u0001\u0000\u0000\u0000\u025f\u0260\u0001\u0000\u0000"+ - "\u0000\u0260\u026a\u0001\u0000\u0000\u0000\u0261\u0262\u0005`\u0000\u0000"+ - "\u0262\u0267\u0003z=\u0000\u0263\u0264\u0005\'\u0000\u0000\u0264\u0266"+ - "\u0003z=\u0000\u0265\u0263\u0001\u0000\u0000\u0000\u0266\u0269\u0001\u0000"+ - "\u0000\u0000\u0267\u0265\u0001\u0000\u0000\u0000\u0267\u0268\u0001\u0000"+ - "\u0000\u0000\u0268\u026b\u0001\u0000\u0000\u0000\u0269\u0267\u0001\u0000"+ - "\u0000\u0000\u026a\u0261\u0001\u0000\u0000\u0000\u026a\u026b\u0001\u0000"+ - "\u0000\u0000\u026by\u0001\u0000\u0000\u0000\u026c\u026d\u0003@ \u0000"+ - "\u026d\u026e\u0005$\u0000\u0000\u026e\u0270\u0001\u0000\u0000\u0000\u026f"+ - "\u026c\u0001\u0000\u0000\u0000\u026f\u0270\u0001\u0000\u0000\u0000\u0270"+ - "\u0271\u0001\u0000\u0000\u0000\u0271\u0272\u0003@ \u0000\u0272{\u0001"+ - "\u0000\u0000\u0000\u0273\u0274\u0005\u0012\u0000\u0000\u0274\u0275\u0003"+ - "(\u0014\u0000\u0275\u0276\u0005_\u0000\u0000\u0276\u0277\u0003B!\u0000"+ - "\u0277}\u0001\u0000\u0000\u0000\u0278\u0279\u0005\u0011\u0000\u0000\u0279"+ - "\u027c\u0003:\u001d\u0000\u027a\u027b\u0005!\u0000\u0000\u027b\u027d\u0003"+ - "\"\u0011\u0000\u027c\u027a\u0001\u0000\u0000\u0000\u027c\u027d\u0001\u0000"+ - "\u0000\u0000\u027d\u007f\u0001\u0000\u0000\u0000\u027e\u0280\u0007\b\u0000"+ - "\u0000\u027f\u027e\u0001\u0000\u0000\u0000\u027f\u0280\u0001\u0000\u0000"+ - "\u0000\u0280\u0281\u0001\u0000\u0000\u0000\u0281\u0282\u0005\u0014\u0000"+ - "\u0000\u0282\u0283\u0003\u0082A\u0000\u0283\u0284\u0003\u0084B\u0000\u0284"+ - "\u0081\u0001\u0000\u0000\u0000\u0285\u0288\u0003(\u0014\u0000\u0286\u0287"+ - "\u0005[\u0000\u0000\u0287\u0289\u0003D\"\u0000\u0288\u0286\u0001\u0000"+ - "\u0000\u0000\u0288\u0289\u0001\u0000\u0000\u0000\u0289\u0083\u0001\u0000"+ - "\u0000\u0000\u028a\u028b\u0005_\u0000\u0000\u028b\u0290\u0003\u0086C\u0000"+ - "\u028c\u028d\u0005\'\u0000\u0000\u028d\u028f\u0003\u0086C\u0000\u028e"+ - "\u028c\u0001\u0000\u0000\u0000\u028f\u0292\u0001\u0000\u0000\u0000\u0290"+ - "\u028e\u0001\u0000\u0000\u0000\u0290\u0291\u0001\u0000\u0000\u0000\u0291"+ - "\u0085\u0001\u0000\u0000\u0000\u0292\u0290\u0001\u0000\u0000\u0000\u0293"+ - "\u0294\u0003\u0010\b\u0000\u0294\u0087\u0001\u0000\u0000\u0000@\u0093"+ - "\u009c\u00b0\u00bc\u00c5\u00cd\u00d2\u00da\u00dc\u00e1\u00e8\u00ed\u00f2"+ - "\u00fc\u0102\u010a\u010c\u0117\u011e\u0129\u012e\u0130\u013c\u014f\u0155"+ - "\u015f\u0163\u0168\u0172\u017a\u0187\u018b\u018f\u0196\u019a\u01a1\u01a7"+ - "\u01ae\u01b6\u01be\u01c6\u01d7\u01e2\u01ed\u01f2\u01f6\u01fb\u0206\u020b"+ - "\u020f\u021d\u0228\u0236\u0241\u0244\u0249\u025f\u0267\u026a\u026f\u027c"+ - "\u027f\u0288\u0290"; + "\u001c\u0196\b\u001c\u0001\u001c\u0001\u001c\u0003\u001c\u019a\b\u001c"+ + "\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019f\b\u001d\n\u001d"+ + "\f\u001d\u01a2\t\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0003\u001e"+ + "\u01a7\b\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01ac\b"+ + "\u001f\n\u001f\f\u001f\u01af\t\u001f\u0001 \u0001 \u0001 \u0005 \u01b4"+ + "\b \n \f \u01b7\t \u0001!\u0001!\u0001!\u0005!\u01bc\b!\n!\f!\u01bf\t"+ + "!\u0001\"\u0001\"\u0001#\u0001#\u0003#\u01c5\b#\u0001$\u0001$\u0001$\u0001"+ + "$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0005"+ + "$\u01d4\b$\n$\f$\u01d7\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0005"+ + "$\u01df\b$\n$\f$\u01e2\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0005"+ + "$\u01ea\b$\n$\f$\u01ed\t$\u0001$\u0001$\u0003$\u01f1\b$\u0001%\u0001%"+ + "\u0003%\u01f5\b%\u0001&\u0001&\u0003&\u01f9\b&\u0001\'\u0001\'\u0001\'"+ + "\u0001(\u0001(\u0001(\u0001(\u0005(\u0202\b(\n(\f(\u0205\t(\u0001)\u0001"+ + ")\u0003)\u0209\b)\u0001)\u0001)\u0003)\u020d\b)\u0001*\u0001*\u0001*\u0001"+ + "+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0005,\u0219\b,\n,\f,\u021c"+ + "\t,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0003.\u0226"+ + "\b.\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u00010\u00011\u00011\u0001"+ + "1\u00051\u0232\b1\n1\f1\u0235\t1\u00012\u00012\u00012\u00012\u00013\u0001"+ + "3\u00014\u00014\u00034\u023f\b4\u00015\u00035\u0242\b5\u00015\u00015\u0001"+ + "6\u00036\u0247\b6\u00016\u00016\u00017\u00017\u00018\u00018\u00019\u0001"+ + "9\u00019\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001<\u0001"+ + "<\u0001<\u0001<\u0003<\u025d\b<\u0001<\u0001<\u0001<\u0001<\u0005<\u0263"+ + "\b<\n<\f<\u0266\t<\u0003<\u0268\b<\u0001=\u0001=\u0001=\u0003=\u026d\b"+ + "=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001>\u0001>\u0001?\u0001?\u0001"+ + "?\u0001?\u0003?\u027a\b?\u0001@\u0001@\u0001@\u0001@\u0001@\u0001A\u0001"+ + "A\u0001B\u0001B\u0001B\u0001B\u0005B\u0287\bB\nB\fB\u028a\tB\u0001C\u0001"+ + "C\u0001C\u0000\u0004\u0002\n\u0012\u0014D\u0000\u0002\u0004\u0006\b\n"+ + "\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.0246"+ + "8:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082\u0084\u0086\u0000\t"+ + "\u0001\u0000?@\u0001\u0000AC\u0002\u0000\u001d\u001dRR\u0001\u0000IJ\u0002"+ + "\u0000\"\"\'\'\u0002\u0000**--\u0002\u0000))77\u0002\u000088:>\u0002\u0000"+ + "\u0011\u0011\u0016\u0017\u02a6\u0000\u0088\u0001\u0000\u0000\u0000\u0002"+ + "\u008b\u0001\u0000\u0000\u0000\u0004\u009c\u0001\u0000\u0000\u0000\u0006"+ + "\u00af\u0001\u0000\u0000\u0000\b\u00b1\u0001\u0000\u0000\u0000\n\u00d1"+ + "\u0001\u0000\u0000\u0000\f\u00ec\u0001\u0000\u0000\u0000\u000e\u00ee\u0001"+ + "\u0000\u0000\u0000\u0010\u00fb\u0001\u0000\u0000\u0000\u0012\u0101\u0001"+ + "\u0000\u0000\u0000\u0014\u0116\u0001\u0000\u0000\u0000\u0016\u0120\u0001"+ + "\u0000\u0000\u0000\u0018\u0133\u0001\u0000\u0000\u0000\u001a\u0135\u0001"+ + "\u0000\u0000\u0000\u001c\u0140\u0001\u0000\u0000\u0000\u001e\u0144\u0001"+ + "\u0000\u0000\u0000 \u0146\u0001\u0000\u0000\u0000\"\u0149\u0001\u0000"+ + "\u0000\u0000$\u0154\u0001\u0000\u0000\u0000&\u0158\u0001\u0000\u0000\u0000"+ + "(\u0167\u0001\u0000\u0000\u0000*\u016b\u0001\u0000\u0000\u0000,\u016d"+ + "\u0001\u0000\u0000\u0000.\u0171\u0001\u0000\u0000\u00000\u0173\u0001\u0000"+ + "\u0000\u00002\u017c\u0001\u0000\u0000\u00004\u0180\u0001\u0000\u0000\u0000"+ + "6\u0190\u0001\u0000\u0000\u00008\u0193\u0001\u0000\u0000\u0000:\u019b"+ + "\u0001\u0000\u0000\u0000<\u01a3\u0001\u0000\u0000\u0000>\u01a8\u0001\u0000"+ + "\u0000\u0000@\u01b0\u0001\u0000\u0000\u0000B\u01b8\u0001\u0000\u0000\u0000"+ + "D\u01c0\u0001\u0000\u0000\u0000F\u01c4\u0001\u0000\u0000\u0000H\u01f0"+ + "\u0001\u0000\u0000\u0000J\u01f4\u0001\u0000\u0000\u0000L\u01f8\u0001\u0000"+ + "\u0000\u0000N\u01fa\u0001\u0000\u0000\u0000P\u01fd\u0001\u0000\u0000\u0000"+ + "R\u0206\u0001\u0000\u0000\u0000T\u020e\u0001\u0000\u0000\u0000V\u0211"+ + "\u0001\u0000\u0000\u0000X\u0214\u0001\u0000\u0000\u0000Z\u021d\u0001\u0000"+ + "\u0000\u0000\\\u0221\u0001\u0000\u0000\u0000^\u0227\u0001\u0000\u0000"+ + "\u0000`\u022b\u0001\u0000\u0000\u0000b\u022e\u0001\u0000\u0000\u0000d"+ + "\u0236\u0001\u0000\u0000\u0000f\u023a\u0001\u0000\u0000\u0000h\u023e\u0001"+ + "\u0000\u0000\u0000j\u0241\u0001\u0000\u0000\u0000l\u0246\u0001\u0000\u0000"+ + "\u0000n\u024a\u0001\u0000\u0000\u0000p\u024c\u0001\u0000\u0000\u0000r"+ + "\u024e\u0001\u0000\u0000\u0000t\u0251\u0001\u0000\u0000\u0000v\u0255\u0001"+ + "\u0000\u0000\u0000x\u0258\u0001\u0000\u0000\u0000z\u026c\u0001\u0000\u0000"+ + "\u0000|\u0270\u0001\u0000\u0000\u0000~\u0275\u0001\u0000\u0000\u0000\u0080"+ + "\u027b\u0001\u0000\u0000\u0000\u0082\u0280\u0001\u0000\u0000\u0000\u0084"+ + "\u0282\u0001\u0000\u0000\u0000\u0086\u028b\u0001\u0000\u0000\u0000\u0088"+ + "\u0089\u0003\u0002\u0001\u0000\u0089\u008a\u0005\u0000\u0000\u0001\u008a"+ + "\u0001\u0001\u0000\u0000\u0000\u008b\u008c\u0006\u0001\uffff\uffff\u0000"+ + "\u008c\u008d\u0003\u0004\u0002\u0000\u008d\u0093\u0001\u0000\u0000\u0000"+ + "\u008e\u008f\n\u0001\u0000\u0000\u008f\u0090\u0005\u001c\u0000\u0000\u0090"+ + "\u0092\u0003\u0006\u0003\u0000\u0091\u008e\u0001\u0000\u0000\u0000\u0092"+ + "\u0095\u0001\u0000\u0000\u0000\u0093\u0091\u0001\u0000\u0000\u0000\u0093"+ + "\u0094\u0001\u0000\u0000\u0000\u0094\u0003\u0001\u0000\u0000\u0000\u0095"+ + "\u0093\u0001\u0000\u0000\u0000\u0096\u009d\u0003r9\u0000\u0097\u009d\u0003"+ + "&\u0013\u0000\u0098\u009d\u0003 \u0010\u0000\u0099\u009d\u0003v;\u0000"+ + "\u009a\u009b\u0004\u0002\u0001\u0000\u009b\u009d\u00034\u001a\u0000\u009c"+ + "\u0096\u0001\u0000\u0000\u0000\u009c\u0097\u0001\u0000\u0000\u0000\u009c"+ + "\u0098\u0001\u0000\u0000\u0000\u009c\u0099\u0001\u0000\u0000\u0000\u009c"+ + "\u009a\u0001\u0000\u0000\u0000\u009d\u0005\u0001\u0000\u0000\u0000\u009e"+ + "\u00b0\u00036\u001b\u0000\u009f\u00b0\u0003\b\u0004\u0000\u00a0\u00b0"+ + "\u0003T*\u0000\u00a1\u00b0\u0003N\'\u0000\u00a2\u00b0\u00038\u001c\u0000"+ + "\u00a3\u00b0\u0003P(\u0000\u00a4\u00b0\u0003V+\u0000\u00a5\u00b0\u0003"+ + "X,\u0000\u00a6\u00b0\u0003\\.\u0000\u00a7\u00b0\u0003^/\u0000\u00a8\u00b0"+ + "\u0003x<\u0000\u00a9\u00b0\u0003`0\u0000\u00aa\u00b0\u0003\u0080@\u0000"+ + "\u00ab\u00ac\u0004\u0003\u0002\u0000\u00ac\u00b0\u0003~?\u0000\u00ad\u00ae"+ + "\u0004\u0003\u0003\u0000\u00ae\u00b0\u0003|>\u0000\u00af\u009e\u0001\u0000"+ + "\u0000\u0000\u00af\u009f\u0001\u0000\u0000\u0000\u00af\u00a0\u0001\u0000"+ + "\u0000\u0000\u00af\u00a1\u0001\u0000\u0000\u0000\u00af\u00a2\u0001\u0000"+ + "\u0000\u0000\u00af\u00a3\u0001\u0000\u0000\u0000\u00af\u00a4\u0001\u0000"+ + "\u0000\u0000\u00af\u00a5\u0001\u0000\u0000\u0000\u00af\u00a6\u0001\u0000"+ + "\u0000\u0000\u00af\u00a7\u0001\u0000\u0000\u0000\u00af\u00a8\u0001\u0000"+ + "\u0000\u0000\u00af\u00a9\u0001\u0000\u0000\u0000\u00af\u00aa\u0001\u0000"+ + "\u0000\u0000\u00af\u00ab\u0001\u0000\u0000\u0000\u00af\u00ad\u0001\u0000"+ + "\u0000\u0000\u00b0\u0007\u0001\u0000\u0000\u0000\u00b1\u00b2\u0005\u0010"+ + "\u0000\u0000\u00b2\u00b3\u0003\n\u0005\u0000\u00b3\t\u0001\u0000\u0000"+ + "\u0000\u00b4\u00b5\u0006\u0005\uffff\uffff\u0000\u00b5\u00b6\u00050\u0000"+ + "\u0000\u00b6\u00d2\u0003\n\u0005\b\u00b7\u00d2\u0003\u0010\b\u0000\u00b8"+ + "\u00d2\u0003\f\u0006\u0000\u00b9\u00bb\u0003\u0010\b\u0000\u00ba\u00bc"+ + "\u00050\u0000\u0000\u00bb\u00ba\u0001\u0000\u0000\u0000\u00bb\u00bc\u0001"+ + "\u0000\u0000\u0000\u00bc\u00bd\u0001\u0000\u0000\u0000\u00bd\u00be\u0005"+ + "+\u0000\u0000\u00be\u00bf\u0005/\u0000\u0000\u00bf\u00c4\u0003\u0010\b"+ + "\u0000\u00c0\u00c1\u0005&\u0000\u0000\u00c1\u00c3\u0003\u0010\b\u0000"+ + "\u00c2\u00c0\u0001\u0000\u0000\u0000\u00c3\u00c6\u0001\u0000\u0000\u0000"+ + "\u00c4\u00c2\u0001\u0000\u0000\u0000\u00c4\u00c5\u0001\u0000\u0000\u0000"+ + "\u00c5\u00c7\u0001\u0000\u0000\u0000\u00c6\u00c4\u0001\u0000\u0000\u0000"+ + "\u00c7\u00c8\u00056\u0000\u0000\u00c8\u00d2\u0001\u0000\u0000\u0000\u00c9"+ + "\u00ca\u0003\u0010\b\u0000\u00ca\u00cc\u0005,\u0000\u0000\u00cb\u00cd"+ + "\u00050\u0000\u0000\u00cc\u00cb\u0001\u0000\u0000\u0000\u00cc\u00cd\u0001"+ + "\u0000\u0000\u0000\u00cd\u00ce\u0001\u0000\u0000\u0000\u00ce\u00cf\u0005"+ + "1\u0000\u0000\u00cf\u00d2\u0001\u0000\u0000\u0000\u00d0\u00d2\u0003\u000e"+ + "\u0007\u0000\u00d1\u00b4\u0001\u0000\u0000\u0000\u00d1\u00b7\u0001\u0000"+ + "\u0000\u0000\u00d1\u00b8\u0001\u0000\u0000\u0000\u00d1\u00b9\u0001\u0000"+ + "\u0000\u0000\u00d1\u00c9\u0001\u0000\u0000\u0000\u00d1\u00d0\u0001\u0000"+ + "\u0000\u0000\u00d2\u00db\u0001\u0000\u0000\u0000\u00d3\u00d4\n\u0005\u0000"+ + "\u0000\u00d4\u00d5\u0005!\u0000\u0000\u00d5\u00da\u0003\n\u0005\u0006"+ + "\u00d6\u00d7\n\u0004\u0000\u0000\u00d7\u00d8\u00053\u0000\u0000\u00d8"+ + "\u00da\u0003\n\u0005\u0005\u00d9\u00d3\u0001\u0000\u0000\u0000\u00d9\u00d6"+ + "\u0001\u0000\u0000\u0000\u00da\u00dd\u0001\u0000\u0000\u0000\u00db\u00d9"+ + "\u0001\u0000\u0000\u0000\u00db\u00dc\u0001\u0000\u0000\u0000\u00dc\u000b"+ + "\u0001\u0000\u0000\u0000\u00dd\u00db\u0001\u0000\u0000\u0000\u00de\u00e0"+ + "\u0003\u0010\b\u0000\u00df\u00e1\u00050\u0000\u0000\u00e0\u00df\u0001"+ + "\u0000\u0000\u0000\u00e0\u00e1\u0001\u0000\u0000\u0000\u00e1\u00e2\u0001"+ + "\u0000\u0000\u0000\u00e2\u00e3\u0005.\u0000\u0000\u00e3\u00e4\u0003n7"+ + "\u0000\u00e4\u00ed\u0001\u0000\u0000\u0000\u00e5\u00e7\u0003\u0010\b\u0000"+ + "\u00e6\u00e8\u00050\u0000\u0000\u00e7\u00e6\u0001\u0000\u0000\u0000\u00e7"+ + "\u00e8\u0001\u0000\u0000\u0000\u00e8\u00e9\u0001\u0000\u0000\u0000\u00e9"+ + "\u00ea\u00055\u0000\u0000\u00ea\u00eb\u0003n7\u0000\u00eb\u00ed\u0001"+ + "\u0000\u0000\u0000\u00ec\u00de\u0001\u0000\u0000\u0000\u00ec\u00e5\u0001"+ + "\u0000\u0000\u0000\u00ed\r\u0001\u0000\u0000\u0000\u00ee\u00f1\u0003>"+ + "\u001f\u0000\u00ef\u00f0\u0005$\u0000\u0000\u00f0\u00f2\u0003\u001e\u000f"+ + "\u0000\u00f1\u00ef\u0001\u0000\u0000\u0000\u00f1\u00f2\u0001\u0000\u0000"+ + "\u0000\u00f2\u00f3\u0001\u0000\u0000\u0000\u00f3\u00f4\u0005%\u0000\u0000"+ + "\u00f4\u00f5\u0003H$\u0000\u00f5\u000f\u0001\u0000\u0000\u0000\u00f6\u00fc"+ + "\u0003\u0012\t\u0000\u00f7\u00f8\u0003\u0012\t\u0000\u00f8\u00f9\u0003"+ + "p8\u0000\u00f9\u00fa\u0003\u0012\t\u0000\u00fa\u00fc\u0001\u0000\u0000"+ + "\u0000\u00fb\u00f6\u0001\u0000\u0000\u0000\u00fb\u00f7\u0001\u0000\u0000"+ + "\u0000\u00fc\u0011\u0001\u0000\u0000\u0000\u00fd\u00fe\u0006\t\uffff\uffff"+ + "\u0000\u00fe\u0102\u0003\u0014\n\u0000\u00ff\u0100\u0007\u0000\u0000\u0000"+ + "\u0100\u0102\u0003\u0012\t\u0003\u0101\u00fd\u0001\u0000\u0000\u0000\u0101"+ + "\u00ff\u0001\u0000\u0000\u0000\u0102\u010b\u0001\u0000\u0000\u0000\u0103"+ + "\u0104\n\u0002\u0000\u0000\u0104\u0105\u0007\u0001\u0000\u0000\u0105\u010a"+ + "\u0003\u0012\t\u0003\u0106\u0107\n\u0001\u0000\u0000\u0107\u0108\u0007"+ + "\u0000\u0000\u0000\u0108\u010a\u0003\u0012\t\u0002\u0109\u0103\u0001\u0000"+ + "\u0000\u0000\u0109\u0106\u0001\u0000\u0000\u0000\u010a\u010d\u0001\u0000"+ + "\u0000\u0000\u010b\u0109\u0001\u0000\u0000\u0000\u010b\u010c\u0001\u0000"+ + "\u0000\u0000\u010c\u0013\u0001\u0000\u0000\u0000\u010d\u010b\u0001\u0000"+ + "\u0000\u0000\u010e\u010f\u0006\n\uffff\uffff\u0000\u010f\u0117\u0003H"+ + "$\u0000\u0110\u0117\u0003>\u001f\u0000\u0111\u0117\u0003\u0016\u000b\u0000"+ + "\u0112\u0113\u0005/\u0000\u0000\u0113\u0114\u0003\n\u0005\u0000\u0114"+ + "\u0115\u00056\u0000\u0000\u0115\u0117\u0001\u0000\u0000\u0000\u0116\u010e"+ + "\u0001\u0000\u0000\u0000\u0116\u0110\u0001\u0000\u0000\u0000\u0116\u0111"+ + "\u0001\u0000\u0000\u0000\u0116\u0112\u0001\u0000\u0000\u0000\u0117\u011d"+ + "\u0001\u0000\u0000\u0000\u0118\u0119\n\u0001\u0000\u0000\u0119\u011a\u0005"+ + "$\u0000\u0000\u011a\u011c\u0003\u001e\u000f\u0000\u011b\u0118\u0001\u0000"+ + "\u0000\u0000\u011c\u011f\u0001\u0000\u0000\u0000\u011d\u011b\u0001\u0000"+ + "\u0000\u0000\u011d\u011e\u0001\u0000\u0000\u0000\u011e\u0015\u0001\u0000"+ + "\u0000\u0000\u011f\u011d\u0001\u0000\u0000\u0000\u0120\u0121\u0003\u0018"+ + "\f\u0000\u0121\u012f\u0005/\u0000\u0000\u0122\u0130\u0005A\u0000\u0000"+ + "\u0123\u0128\u0003\n\u0005\u0000\u0124\u0125\u0005&\u0000\u0000\u0125"+ + "\u0127\u0003\n\u0005\u0000\u0126\u0124\u0001\u0000\u0000\u0000\u0127\u012a"+ + "\u0001\u0000\u0000\u0000\u0128\u0126\u0001\u0000\u0000\u0000\u0128\u0129"+ + "\u0001\u0000\u0000\u0000\u0129\u012d\u0001\u0000\u0000\u0000\u012a\u0128"+ + "\u0001\u0000\u0000\u0000\u012b\u012c\u0005&\u0000\u0000\u012c\u012e\u0003"+ + "\u001a\r\u0000\u012d\u012b\u0001\u0000\u0000\u0000\u012d\u012e\u0001\u0000"+ + "\u0000\u0000\u012e\u0130\u0001\u0000\u0000\u0000\u012f\u0122\u0001\u0000"+ + "\u0000\u0000\u012f\u0123\u0001\u0000\u0000\u0000\u012f\u0130\u0001\u0000"+ + "\u0000\u0000\u0130\u0131\u0001\u0000\u0000\u0000\u0131\u0132\u00056\u0000"+ + "\u0000\u0132\u0017\u0001\u0000\u0000\u0000\u0133\u0134\u0003L&\u0000\u0134"+ + "\u0019\u0001\u0000\u0000\u0000\u0135\u0136\u0005D\u0000\u0000\u0136\u013b"+ + "\u0003\u001c\u000e\u0000\u0137\u0138\u0005&\u0000\u0000\u0138\u013a\u0003"+ + "\u001c\u000e\u0000\u0139\u0137\u0001\u0000\u0000\u0000\u013a\u013d\u0001"+ + "\u0000\u0000\u0000\u013b\u0139\u0001\u0000\u0000\u0000\u013b\u013c\u0001"+ + "\u0000\u0000\u0000\u013c\u013e\u0001\u0000\u0000\u0000\u013d\u013b\u0001"+ + "\u0000\u0000\u0000\u013e\u013f\u0005E\u0000\u0000\u013f\u001b\u0001\u0000"+ + "\u0000\u0000\u0140\u0141\u0003n7\u0000\u0141\u0142\u0005%\u0000\u0000"+ + "\u0142\u0143\u0003H$\u0000\u0143\u001d\u0001\u0000\u0000\u0000\u0144\u0145"+ + "\u0003D\"\u0000\u0145\u001f\u0001\u0000\u0000\u0000\u0146\u0147\u0005"+ + "\f\u0000\u0000\u0147\u0148\u0003\"\u0011\u0000\u0148!\u0001\u0000\u0000"+ + "\u0000\u0149\u014e\u0003$\u0012\u0000\u014a\u014b\u0005&\u0000\u0000\u014b"+ + "\u014d\u0003$\u0012\u0000\u014c\u014a\u0001\u0000\u0000\u0000\u014d\u0150"+ + "\u0001\u0000\u0000\u0000\u014e\u014c\u0001\u0000\u0000\u0000\u014e\u014f"+ + "\u0001\u0000\u0000\u0000\u014f#\u0001\u0000\u0000\u0000\u0150\u014e\u0001"+ + "\u0000\u0000\u0000\u0151\u0152\u0003>\u001f\u0000\u0152\u0153\u0005#\u0000"+ + "\u0000\u0153\u0155\u0001\u0000\u0000\u0000\u0154\u0151\u0001\u0000\u0000"+ + "\u0000\u0154\u0155\u0001\u0000\u0000\u0000\u0155\u0156\u0001\u0000\u0000"+ + "\u0000\u0156\u0157\u0003\n\u0005\u0000\u0157%\u0001\u0000\u0000\u0000"+ + "\u0158\u0159\u0005\u0006\u0000\u0000\u0159\u015e\u0003(\u0014\u0000\u015a"+ + "\u015b\u0005&\u0000\u0000\u015b\u015d\u0003(\u0014\u0000\u015c\u015a\u0001"+ + "\u0000\u0000\u0000\u015d\u0160\u0001\u0000\u0000\u0000\u015e\u015c\u0001"+ + "\u0000\u0000\u0000\u015e\u015f\u0001\u0000\u0000\u0000\u015f\u0162\u0001"+ + "\u0000\u0000\u0000\u0160\u015e\u0001\u0000\u0000\u0000\u0161\u0163\u0003"+ + ".\u0017\u0000\u0162\u0161\u0001\u0000\u0000\u0000\u0162\u0163\u0001\u0000"+ + "\u0000\u0000\u0163\'\u0001\u0000\u0000\u0000\u0164\u0165\u0003*\u0015"+ + "\u0000\u0165\u0166\u0005%\u0000\u0000\u0166\u0168\u0001\u0000\u0000\u0000"+ + "\u0167\u0164\u0001\u0000\u0000\u0000\u0167\u0168\u0001\u0000\u0000\u0000"+ + "\u0168\u0169\u0001\u0000\u0000\u0000\u0169\u016a\u0003,\u0016\u0000\u016a"+ + ")\u0001\u0000\u0000\u0000\u016b\u016c\u0005R\u0000\u0000\u016c+\u0001"+ + "\u0000\u0000\u0000\u016d\u016e\u0007\u0002\u0000\u0000\u016e-\u0001\u0000"+ + "\u0000\u0000\u016f\u0172\u00030\u0018\u0000\u0170\u0172\u00032\u0019\u0000"+ + "\u0171\u016f\u0001\u0000\u0000\u0000\u0171\u0170\u0001\u0000\u0000\u0000"+ + "\u0172/\u0001\u0000\u0000\u0000\u0173\u0174\u0005Q\u0000\u0000\u0174\u0179"+ + "\u0005R\u0000\u0000\u0175\u0176\u0005&\u0000\u0000\u0176\u0178\u0005R"+ + "\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178\u017b\u0001\u0000"+ + "\u0000\u0000\u0179\u0177\u0001\u0000\u0000\u0000\u0179\u017a\u0001\u0000"+ + "\u0000\u0000\u017a1\u0001\u0000\u0000\u0000\u017b\u0179\u0001\u0000\u0000"+ + "\u0000\u017c\u017d\u0005G\u0000\u0000\u017d\u017e\u00030\u0018\u0000\u017e"+ + "\u017f\u0005H\u0000\u0000\u017f3\u0001\u0000\u0000\u0000\u0180\u0181\u0005"+ + "\u0014\u0000\u0000\u0181\u0186\u0003(\u0014\u0000\u0182\u0183\u0005&\u0000"+ + "\u0000\u0183\u0185\u0003(\u0014\u0000\u0184\u0182\u0001\u0000\u0000\u0000"+ + "\u0185\u0188\u0001\u0000\u0000\u0000\u0186\u0184\u0001\u0000\u0000\u0000"+ + "\u0186\u0187\u0001\u0000\u0000\u0000\u0187\u018a\u0001\u0000\u0000\u0000"+ + "\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018b\u0003:\u001d\u0000\u018a"+ + "\u0189\u0001\u0000\u0000\u0000\u018a\u018b\u0001\u0000\u0000\u0000\u018b"+ + "\u018e\u0001\u0000\u0000\u0000\u018c\u018d\u0005 \u0000\u0000\u018d\u018f"+ + "\u0003\"\u0011\u0000\u018e\u018c\u0001\u0000\u0000\u0000\u018e\u018f\u0001"+ + "\u0000\u0000\u0000\u018f5\u0001\u0000\u0000\u0000\u0190\u0191\u0005\u0004"+ + "\u0000\u0000\u0191\u0192\u0003\"\u0011\u0000\u01927\u0001\u0000\u0000"+ + "\u0000\u0193\u0195\u0005\u000f\u0000\u0000\u0194\u0196\u0003:\u001d\u0000"+ + "\u0195\u0194\u0001\u0000\u0000\u0000\u0195\u0196\u0001\u0000\u0000\u0000"+ + "\u0196\u0199\u0001\u0000\u0000\u0000\u0197\u0198\u0005 \u0000\u0000\u0198"+ + "\u019a\u0003\"\u0011\u0000\u0199\u0197\u0001\u0000\u0000\u0000\u0199\u019a"+ + "\u0001\u0000\u0000\u0000\u019a9\u0001\u0000\u0000\u0000\u019b\u01a0\u0003"+ + "<\u001e\u0000\u019c\u019d\u0005&\u0000\u0000\u019d\u019f\u0003<\u001e"+ + "\u0000\u019e\u019c\u0001\u0000\u0000\u0000\u019f\u01a2\u0001\u0000\u0000"+ + "\u0000\u01a0\u019e\u0001\u0000\u0000\u0000\u01a0\u01a1\u0001\u0000\u0000"+ + "\u0000\u01a1;\u0001\u0000\u0000\u0000\u01a2\u01a0\u0001\u0000\u0000\u0000"+ + "\u01a3\u01a6\u0003$\u0012\u0000\u01a4\u01a5\u0005\u0010\u0000\u0000\u01a5"+ + "\u01a7\u0003\n\u0005\u0000\u01a6\u01a4\u0001\u0000\u0000\u0000\u01a6\u01a7"+ + "\u0001\u0000\u0000\u0000\u01a7=\u0001\u0000\u0000\u0000\u01a8\u01ad\u0003"+ + "L&\u0000\u01a9\u01aa\u0005(\u0000\u0000\u01aa\u01ac\u0003L&\u0000\u01ab"+ + "\u01a9\u0001\u0000\u0000\u0000\u01ac\u01af\u0001\u0000\u0000\u0000\u01ad"+ + "\u01ab\u0001\u0000\u0000\u0000\u01ad\u01ae\u0001\u0000\u0000\u0000\u01ae"+ + "?\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01b0\u01b5"+ + "\u0003F#\u0000\u01b1\u01b2\u0005(\u0000\u0000\u01b2\u01b4\u0003F#\u0000"+ + "\u01b3\u01b1\u0001\u0000\u0000\u0000\u01b4\u01b7\u0001\u0000\u0000\u0000"+ + "\u01b5\u01b3\u0001\u0000\u0000\u0000\u01b5\u01b6\u0001\u0000\u0000\u0000"+ + "\u01b6A\u0001\u0000\u0000\u0000\u01b7\u01b5\u0001\u0000\u0000\u0000\u01b8"+ + "\u01bd\u0003@ \u0000\u01b9\u01ba\u0005&\u0000\u0000\u01ba\u01bc\u0003"+ + "@ \u0000\u01bb\u01b9\u0001\u0000\u0000\u0000\u01bc\u01bf\u0001\u0000\u0000"+ + "\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01bd\u01be\u0001\u0000\u0000"+ + "\u0000\u01beC\u0001\u0000\u0000\u0000\u01bf\u01bd\u0001\u0000\u0000\u0000"+ + "\u01c0\u01c1\u0007\u0003\u0000\u0000\u01c1E\u0001\u0000\u0000\u0000\u01c2"+ + "\u01c5\u0005V\u0000\u0000\u01c3\u01c5\u0003J%\u0000\u01c4\u01c2\u0001"+ + "\u0000\u0000\u0000\u01c4\u01c3\u0001\u0000\u0000\u0000\u01c5G\u0001\u0000"+ + "\u0000\u0000\u01c6\u01f1\u00051\u0000\u0000\u01c7\u01c8\u0003l6\u0000"+ + "\u01c8\u01c9\u0005I\u0000\u0000\u01c9\u01f1\u0001\u0000\u0000\u0000\u01ca"+ + "\u01f1\u0003j5\u0000\u01cb\u01f1\u0003l6\u0000\u01cc\u01f1\u0003f3\u0000"+ + "\u01cd\u01f1\u0003J%\u0000\u01ce\u01f1\u0003n7\u0000\u01cf\u01d0\u0005"+ + "G\u0000\u0000\u01d0\u01d5\u0003h4\u0000\u01d1\u01d2\u0005&\u0000\u0000"+ + "\u01d2\u01d4\u0003h4\u0000\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d4\u01d7"+ + "\u0001\u0000\u0000\u0000\u01d5\u01d3\u0001\u0000\u0000\u0000\u01d5\u01d6"+ + "\u0001\u0000\u0000\u0000\u01d6\u01d8\u0001\u0000\u0000\u0000\u01d7\u01d5"+ + "\u0001\u0000\u0000\u0000\u01d8\u01d9\u0005H\u0000\u0000\u01d9\u01f1\u0001"+ + "\u0000\u0000\u0000\u01da\u01db\u0005G\u0000\u0000\u01db\u01e0\u0003f3"+ + "\u0000\u01dc\u01dd\u0005&\u0000\u0000\u01dd\u01df\u0003f3\u0000\u01de"+ + "\u01dc\u0001\u0000\u0000\u0000\u01df\u01e2\u0001\u0000\u0000\u0000\u01e0"+ + "\u01de\u0001\u0000\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1"+ + "\u01e3\u0001\u0000\u0000\u0000\u01e2\u01e0\u0001\u0000\u0000\u0000\u01e3"+ + "\u01e4\u0005H\u0000\u0000\u01e4\u01f1\u0001\u0000\u0000\u0000\u01e5\u01e6"+ + "\u0005G\u0000\u0000\u01e6\u01eb\u0003n7\u0000\u01e7\u01e8\u0005&\u0000"+ + "\u0000\u01e8\u01ea\u0003n7\u0000\u01e9\u01e7\u0001\u0000\u0000\u0000\u01ea"+ + "\u01ed\u0001\u0000\u0000\u0000\u01eb\u01e9\u0001\u0000\u0000\u0000\u01eb"+ + "\u01ec\u0001\u0000\u0000\u0000\u01ec\u01ee\u0001\u0000\u0000\u0000\u01ed"+ + "\u01eb\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005H\u0000\u0000\u01ef\u01f1"+ + "\u0001\u0000\u0000\u0000\u01f0\u01c6\u0001\u0000\u0000\u0000\u01f0\u01c7"+ + "\u0001\u0000\u0000\u0000\u01f0\u01ca\u0001\u0000\u0000\u0000\u01f0\u01cb"+ + "\u0001\u0000\u0000\u0000\u01f0\u01cc\u0001\u0000\u0000\u0000\u01f0\u01cd"+ + "\u0001\u0000\u0000\u0000\u01f0\u01ce\u0001\u0000\u0000\u0000\u01f0\u01cf"+ + "\u0001\u0000\u0000\u0000\u01f0\u01da\u0001\u0000\u0000\u0000\u01f0\u01e5"+ + "\u0001\u0000\u0000\u0000\u01f1I\u0001\u0000\u0000\u0000\u01f2\u01f5\u0005"+ + "4\u0000\u0000\u01f3\u01f5\u0005F\u0000\u0000\u01f4\u01f2\u0001\u0000\u0000"+ + "\u0000\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f5K\u0001\u0000\u0000\u0000"+ + "\u01f6\u01f9\u0003D\"\u0000\u01f7\u01f9\u0003J%\u0000\u01f8\u01f6\u0001"+ + "\u0000\u0000\u0000\u01f8\u01f7\u0001\u0000\u0000\u0000\u01f9M\u0001\u0000"+ + "\u0000\u0000\u01fa\u01fb\u0005\t\u0000\u0000\u01fb\u01fc\u0005\u001e\u0000"+ + "\u0000\u01fcO\u0001\u0000\u0000\u0000\u01fd\u01fe\u0005\u000e\u0000\u0000"+ + "\u01fe\u0203\u0003R)\u0000\u01ff\u0200\u0005&\u0000\u0000\u0200\u0202"+ + "\u0003R)\u0000\u0201\u01ff\u0001\u0000\u0000\u0000\u0202\u0205\u0001\u0000"+ + "\u0000\u0000\u0203\u0201\u0001\u0000\u0000\u0000\u0203\u0204\u0001\u0000"+ + "\u0000\u0000\u0204Q\u0001\u0000\u0000\u0000\u0205\u0203\u0001\u0000\u0000"+ + "\u0000\u0206\u0208\u0003\n\u0005\u0000\u0207\u0209\u0007\u0004\u0000\u0000"+ + "\u0208\u0207\u0001\u0000\u0000\u0000\u0208\u0209\u0001\u0000\u0000\u0000"+ + "\u0209\u020c\u0001\u0000\u0000\u0000\u020a\u020b\u00052\u0000\u0000\u020b"+ + "\u020d\u0007\u0005\u0000\u0000\u020c\u020a\u0001\u0000\u0000\u0000\u020c"+ + "\u020d\u0001\u0000\u0000\u0000\u020dS\u0001\u0000\u0000\u0000\u020e\u020f"+ + "\u0005\b\u0000\u0000\u020f\u0210\u0003B!\u0000\u0210U\u0001\u0000\u0000"+ + "\u0000\u0211\u0212\u0005\u0002\u0000\u0000\u0212\u0213\u0003B!\u0000\u0213"+ + "W\u0001\u0000\u0000\u0000\u0214\u0215\u0005\u000b\u0000\u0000\u0215\u021a"+ + "\u0003Z-\u0000\u0216\u0217\u0005&\u0000\u0000\u0217\u0219\u0003Z-\u0000"+ + "\u0218\u0216\u0001\u0000\u0000\u0000\u0219\u021c\u0001\u0000\u0000\u0000"+ + "\u021a\u0218\u0001\u0000\u0000\u0000\u021a\u021b\u0001\u0000\u0000\u0000"+ + "\u021bY\u0001\u0000\u0000\u0000\u021c\u021a\u0001\u0000\u0000\u0000\u021d"+ + "\u021e\u0003@ \u0000\u021e\u021f\u0005Z\u0000\u0000\u021f\u0220\u0003"+ + "@ \u0000\u0220[\u0001\u0000\u0000\u0000\u0221\u0222\u0005\u0001\u0000"+ + "\u0000\u0222\u0223\u0003\u0014\n\u0000\u0223\u0225\u0003n7\u0000\u0224"+ + "\u0226\u0003b1\u0000\u0225\u0224\u0001\u0000\u0000\u0000\u0225\u0226\u0001"+ + "\u0000\u0000\u0000\u0226]\u0001\u0000\u0000\u0000\u0227\u0228\u0005\u0007"+ + "\u0000\u0000\u0228\u0229\u0003\u0014\n\u0000\u0229\u022a\u0003n7\u0000"+ + "\u022a_\u0001\u0000\u0000\u0000\u022b\u022c\u0005\n\u0000\u0000\u022c"+ + "\u022d\u0003>\u001f\u0000\u022da\u0001\u0000\u0000\u0000\u022e\u0233\u0003"+ + "d2\u0000\u022f\u0230\u0005&\u0000\u0000\u0230\u0232\u0003d2\u0000\u0231"+ + "\u022f\u0001\u0000\u0000\u0000\u0232\u0235\u0001\u0000\u0000\u0000\u0233"+ + "\u0231\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000\u0234"+ + "c\u0001\u0000\u0000\u0000\u0235\u0233\u0001\u0000\u0000\u0000\u0236\u0237"+ + "\u0003D\"\u0000\u0237\u0238\u0005#\u0000\u0000\u0238\u0239\u0003H$\u0000"+ + "\u0239e\u0001\u0000\u0000\u0000\u023a\u023b\u0007\u0006\u0000\u0000\u023b"+ + "g\u0001\u0000\u0000\u0000\u023c\u023f\u0003j5\u0000\u023d\u023f\u0003"+ + "l6\u0000\u023e\u023c\u0001\u0000\u0000\u0000\u023e\u023d\u0001\u0000\u0000"+ + "\u0000\u023fi\u0001\u0000\u0000\u0000\u0240\u0242\u0007\u0000\u0000\u0000"+ + "\u0241\u0240\u0001\u0000\u0000\u0000\u0241\u0242\u0001\u0000\u0000\u0000"+ + "\u0242\u0243\u0001\u0000\u0000\u0000\u0243\u0244\u0005\u001f\u0000\u0000"+ + "\u0244k\u0001\u0000\u0000\u0000\u0245\u0247\u0007\u0000\u0000\u0000\u0246"+ + "\u0245\u0001\u0000\u0000\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0247"+ + "\u0248\u0001\u0000\u0000\u0000\u0248\u0249\u0005\u001e\u0000\u0000\u0249"+ + "m\u0001\u0000\u0000\u0000\u024a\u024b\u0005\u001d\u0000\u0000\u024bo\u0001"+ + "\u0000\u0000\u0000\u024c\u024d\u0007\u0007\u0000\u0000\u024dq\u0001\u0000"+ + "\u0000\u0000\u024e\u024f\u0005\u0005\u0000\u0000\u024f\u0250\u0003t:\u0000"+ + "\u0250s\u0001\u0000\u0000\u0000\u0251\u0252\u0005G\u0000\u0000\u0252\u0253"+ + "\u0003\u0002\u0001\u0000\u0253\u0254\u0005H\u0000\u0000\u0254u\u0001\u0000"+ + "\u0000\u0000\u0255\u0256\u0005\r\u0000\u0000\u0256\u0257\u0005j\u0000"+ + "\u0000\u0257w\u0001\u0000\u0000\u0000\u0258\u0259\u0005\u0003\u0000\u0000"+ + "\u0259\u025c\u0005`\u0000\u0000\u025a\u025b\u0005^\u0000\u0000\u025b\u025d"+ + "\u0003@ \u0000\u025c\u025a\u0001\u0000\u0000\u0000\u025c\u025d\u0001\u0000"+ + "\u0000\u0000\u025d\u0267\u0001\u0000\u0000\u0000\u025e\u025f\u0005_\u0000"+ + "\u0000\u025f\u0264\u0003z=\u0000\u0260\u0261\u0005&\u0000\u0000\u0261"+ + "\u0263\u0003z=\u0000\u0262\u0260\u0001\u0000\u0000\u0000\u0263\u0266\u0001"+ + "\u0000\u0000\u0000\u0264\u0262\u0001\u0000\u0000\u0000\u0264\u0265\u0001"+ + "\u0000\u0000\u0000\u0265\u0268\u0001\u0000\u0000\u0000\u0266\u0264\u0001"+ + "\u0000\u0000\u0000\u0267\u025e\u0001\u0000\u0000\u0000\u0267\u0268\u0001"+ + "\u0000\u0000\u0000\u0268y\u0001\u0000\u0000\u0000\u0269\u026a\u0003@ "+ + "\u0000\u026a\u026b\u0005#\u0000\u0000\u026b\u026d\u0001\u0000\u0000\u0000"+ + "\u026c\u0269\u0001\u0000\u0000\u0000\u026c\u026d\u0001\u0000\u0000\u0000"+ + "\u026d\u026e\u0001\u0000\u0000\u0000\u026e\u026f\u0003@ \u0000\u026f{"+ + "\u0001\u0000\u0000\u0000\u0270\u0271\u0005\u0013\u0000\u0000\u0271\u0272"+ + "\u0003(\u0014\u0000\u0272\u0273\u0005^\u0000\u0000\u0273\u0274\u0003B"+ + "!\u0000\u0274}\u0001\u0000\u0000\u0000\u0275\u0276\u0005\u0012\u0000\u0000"+ + "\u0276\u0279\u0003:\u001d\u0000\u0277\u0278\u0005 \u0000\u0000\u0278\u027a"+ + "\u0003\"\u0011\u0000\u0279\u0277\u0001\u0000\u0000\u0000\u0279\u027a\u0001"+ + "\u0000\u0000\u0000\u027a\u007f\u0001\u0000\u0000\u0000\u027b\u027c\u0007"+ + "\b\u0000\u0000\u027c\u027d\u0005x\u0000\u0000\u027d\u027e\u0003\u0082"+ + "A\u0000\u027e\u027f\u0003\u0084B\u0000\u027f\u0081\u0001\u0000\u0000\u0000"+ + "\u0280\u0281\u0003(\u0014\u0000\u0281\u0083\u0001\u0000\u0000\u0000\u0282"+ + "\u0283\u0005^\u0000\u0000\u0283\u0288\u0003\u0086C\u0000\u0284\u0285\u0005"+ + "&\u0000\u0000\u0285\u0287\u0003\u0086C\u0000\u0286\u0284\u0001\u0000\u0000"+ + "\u0000\u0287\u028a\u0001\u0000\u0000\u0000\u0288\u0286\u0001\u0000\u0000"+ + "\u0000\u0288\u0289\u0001\u0000\u0000\u0000\u0289\u0085\u0001\u0000\u0000"+ + "\u0000\u028a\u0288\u0001\u0000\u0000\u0000\u028b\u028c\u0003\u0010\b\u0000"+ + "\u028c\u0087\u0001\u0000\u0000\u0000>\u0093\u009c\u00af\u00bb\u00c4\u00cc"+ + "\u00d1\u00d9\u00db\u00e0\u00e7\u00ec\u00f1\u00fb\u0101\u0109\u010b\u0116"+ + "\u011d\u0128\u012d\u012f\u013b\u014e\u0154\u015e\u0162\u0167\u0171\u0179"+ + "\u0186\u018a\u018e\u0195\u0199\u01a0\u01a6\u01ad\u01b5\u01bd\u01c4\u01d5"+ + "\u01e0\u01eb\u01f0\u01f4\u01f8\u0203\u0208\u020c\u021a\u0225\u0233\u023e"+ + "\u0241\u0246\u025c\u0264\u0267\u026c\u0279\u0288"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 5912f1fe58bcd..2f12ac8a2f21e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; import java.util.BitSet; +import java.util.EmptyStackException; import java.util.function.BiFunction; import java.util.function.Function; import java.util.regex.Matcher; @@ -111,6 +112,9 @@ private T invokeParser( return result.apply(new AstBuilder(new ExpressionBuilder.ParsingContext(params, metrics)), tree); } catch (StackOverflowError e) { throw new ParsingException("ESQL statement is too large, causing stack overflow when generating the parsing tree: [{}]", query); + // likely thrown by an invalid popMode (such as extra closing parenthesis) + } catch (EmptyStackException ese) { + throw new ParsingException("Invalid query [{}]", query); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 0d09858d33f96..cabd02a07a807 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -795,9 +795,9 @@ public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseCont private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternContext ctx) { return visitQualifiedNamePattern(ctx, ne -> { - if (ne instanceof UnresolvedNamePattern up) { + if (ne instanceof UnresolvedNamePattern || ne instanceof UnresolvedStar) { var src = ne.source(); - throw new ParsingException(src, "Using wildcards [*] in ENRICH WITH projections is not allowed [{}]", up.pattern()); + throw new ParsingException(src, "Using wildcards [*] in ENRICH WITH projections is not allowed, found [{}]", src.text()); } }); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index c5fc929a5a1b8..bd2440e4f38cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -109,17 +110,7 @@ protected LogicalPlan plan(ParseTree ctx) { if (errors.hasNext() == false) { return p; } else { - StringBuilder message = new StringBuilder(); - int i = 0; - - while (errors.hasNext()) { - if (i > 0) { - message.append("; "); - } - message.append(errors.next().getMessage()); - i++; - } - throw new ParsingException(message.toString()); + throw ParsingException.combineParsingExceptions(errors); } } @@ -447,7 +438,11 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { : matchField instanceof UnresolvedStar ? WILDCARD : null; if (patternString != null) { - throw new ParsingException(source, "Using wildcards [*] in ENRICH WITH projections is not allowed [{}]", patternString); + throw new ParsingException( + source, + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [{}]", + patternString + ); } List keepClauses = visitList(this, ctx.enrichWithClause(), NamedExpression.class); @@ -541,11 +536,11 @@ public PlanFactory visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) { public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) { var source = source(ctx); - if (false == Build.current().isSnapshot()) { + if (false == EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) { throw new ParsingException(source, "JOIN is in preview and only available in SNAPSHOT build"); } - if (ctx.type != null && ctx.type.getType() != EsqlBaseParser.DEV_JOIN_LOOKUP) { + if (ctx.type != null && ctx.type.getType() != EsqlBaseParser.JOIN_LOOKUP) { String joinType = ctx.type == null ? "(INNER)" : ctx.type.getText(); throw new ParsingException(source, "only LOOKUP JOIN available, {} JOIN unsupported at the moment", joinType); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java index c25ab92437bfc..119e96bbd865c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java @@ -9,6 +9,8 @@ import org.elasticsearch.xpack.esql.EsqlClientException; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.util.Iterator; + import static org.elasticsearch.common.logging.LoggerMessageFormat.format; public class ParsingException extends EsqlClientException { @@ -21,6 +23,10 @@ public ParsingException(String message, Exception cause, int line, int charPosit this.charPositionInLine = charPositionInLine + 1; } + /** + * To be used only if the exception cannot be associated with a specific position in the query. + * Error message will start with {@code line -1:-1:} instead of using specific location. + */ public ParsingException(String message, Object... args) { this(Source.EMPTY, message, args); } @@ -37,6 +43,38 @@ public ParsingException(Exception cause, Source source, String message, Object.. this.charPositionInLine = source.source().getColumnNumber(); } + private ParsingException(int line, int charPositionInLine, String message, Object... args) { + super(message, args); + this.line = line; + this.charPositionInLine = charPositionInLine; + } + + /** + * Combine multiple {@code ParsingException} into one, this is used by {@code LogicalPlanBuilder} to + * consolidate multiple named parameters related {@code ParsingException}. + */ + public static ParsingException combineParsingExceptions(Iterator parsingExceptions) { + StringBuilder message = new StringBuilder(); + int i = 0; + int line = -1; + int charPositionInLine = -1; + + while (parsingExceptions.hasNext()) { + ParsingException e = parsingExceptions.next(); + if (i > 0) { + message.append("; "); + message.append(e.getMessage()); + } else { + // line and column numbers are the associated with the first error + line = e.getLineNumber(); + charPositionInLine = e.getColumnNumber(); + message.append(e.getErrorMessage()); + } + i++; + } + return new ParsingException(line, charPositionInLine, message.toString()); + } + public int getLineNumber() { return line; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java index b3c273cbfa1bb..a345f69af0247 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/PlanWritables.java @@ -43,7 +43,6 @@ import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec; -import org.elasticsearch.xpack.esql.plan.physical.OrderExec; import org.elasticsearch.xpack.esql.plan.physical.ProjectExec; import org.elasticsearch.xpack.esql.plan.physical.ShowExec; import org.elasticsearch.xpack.esql.plan.physical.SubqueryExec; @@ -103,7 +102,6 @@ public static List phsyical() { LimitExec.ENTRY, LocalSourceExec.ENTRY, MvExpandExec.ENTRY, - OrderExec.ENTRY, ProjectExec.ENTRY, ShowExec.ENTRY, SubqueryExec.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java index 02373cc62e81f..d70177508d847 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java @@ -131,8 +131,8 @@ public PlanType transformExpressionsUp(Class typeToken @SuppressWarnings("unchecked") private static Object doTransformExpression(Object arg, Function traversal) { - if (arg instanceof Expression) { - return traversal.apply((Expression) arg); + if (arg instanceof Expression exp) { + return traversal.apply(exp); } // WARNING: if the collection is typed, an incompatible function will be applied to it @@ -141,17 +141,19 @@ private static Object doTransformExpression(Object arg, Function c) { - List transformed = new ArrayList<>(c.size()); + List transformed = null; boolean hasChanged = false; + int i = 0; for (Object e : c) { Object next = doTransformExpression(e, traversal); - if (e.equals(next)) { - // use the initial value - next = e; - } else { - hasChanged = true; + if (e.equals(next) == false) { + if (hasChanged == false) { + hasChanged = true; + transformed = new ArrayList<>(c); + } + transformed.set(i, next); } - transformed.add(next); + i++; } return hasChanged ? transformed : arg; @@ -186,8 +188,8 @@ public void forEachExpressionUp(Class typeToken, Consu @SuppressWarnings("unchecked") private static void doForEachExpression(Object arg, Consumer traversal) { - if (arg instanceof Expression) { - traversal.accept((Expression) arg); + if (arg instanceof Expression exp) { + traversal.accept(exp); } else if (arg instanceof Collection c) { for (Object o : c) { doForEachExpression(o, traversal); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java index 5c40bfce32064..8cff1d4c88e90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java @@ -40,7 +40,7 @@ import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; import static org.elasticsearch.xpack.esql.plan.logical.Filter.checkFilterConditionDataType; -public class Aggregate extends UnaryPlan implements PostAnalysisVerificationAware, TelemetryAware { +public class Aggregate extends UnaryPlan implements PostAnalysisVerificationAware, TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( LogicalPlan.class, "Aggregate", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java index 483c3508013ab..c8668f58ab5c0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java @@ -17,7 +17,7 @@ import java.util.List; import java.util.Objects; -public class Drop extends UnaryPlan implements TelemetryAware { +public class Drop extends UnaryPlan implements TelemetryAware, SortAgnostic { private final List removals; public Drop(Source source, LogicalPlan child, List removals) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index 4e9fc87318029..11e9a57064e5b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -49,7 +49,7 @@ import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; -public class Enrich extends UnaryPlan implements GeneratingPlan, PostAnalysisPlanVerificationAware, TelemetryAware { +public class Enrich extends UnaryPlan implements GeneratingPlan, PostAnalysisPlanVerificationAware, TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( LogicalPlan.class, "Enrich", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index 448085df1e831..e3c562d3d630e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -226,4 +226,8 @@ public static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws throw new IllegalStateException("not ready to support index mode [" + indexMode + "]"); } } + + public EsRelation withAttributes(List newAttributes) { + return new EsRelation(source(), indexPattern, indexMode, indexNameWithModes, newAttributes); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index 7c437dac03409..af81e26d57c60 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -38,7 +38,7 @@ import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; -public class Eval extends UnaryPlan implements GeneratingPlan, PostAnalysisVerificationAware, TelemetryAware { +public class Eval extends UnaryPlan implements GeneratingPlan, PostAnalysisVerificationAware, TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Eval", Eval::new); private final List fields; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java index 6931c320007fe..7a1726ea59e97 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java @@ -29,7 +29,7 @@ * {@code SELECT x FROM y WHERE z ..} the "WHERE" clause is a Filter. A * {@code Filter} has a "condition" Expression that does the filtering. */ -public class Filter extends UnaryPlan implements PostAnalysisVerificationAware, TelemetryAware { +public class Filter extends UnaryPlan implements PostAnalysisVerificationAware, TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Filter", Filter::new); private final Expression condition; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java index 527ba28d377f1..724aa2da25983 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java @@ -37,7 +37,7 @@ * underlying aggregate. *

*/ -public class InlineStats extends UnaryPlan implements NamedWriteable, SurrogateLogicalPlan, TelemetryAware { +public class InlineStats extends UnaryPlan implements NamedWriteable, SurrogateLogicalPlan, TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( LogicalPlan.class, "InlineStats", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Keep.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Keep.java index 67108afb94668..268c6bbe17242 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Keep.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Keep.java @@ -15,7 +15,7 @@ import java.util.List; import java.util.Objects; -public class Keep extends Project implements TelemetryAware { +public class Keep extends Project implements TelemetryAware, SortAgnostic { public Keep(Source source, LogicalPlan child, List projections) { super(source, child, projections); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java index 1c05ceb124529..56dae7b1f16c0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java @@ -32,7 +32,7 @@ * Looks up values from the associated {@code tables}. * The class is supposed to be substituted by a {@link Join}. */ -public class Lookup extends UnaryPlan implements SurrogateLogicalPlan, TelemetryAware { +public class Lookup extends UnaryPlan implements SurrogateLogicalPlan, TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Lookup", Lookup::new); private final Expression tableName; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index e700ad90afdab..f65811fc26526 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Objects; -public class MvExpand extends UnaryPlan implements TelemetryAware { +public class MvExpand extends UnaryPlan implements TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "MvExpand", MvExpand::new); private final NamedExpression target; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java index 051e2c7769bde..ddb07e0490db3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; @@ -25,7 +26,12 @@ import static org.elasticsearch.xpack.esql.common.Failure.fail; -public class OrderBy extends UnaryPlan implements PostAnalysisVerificationAware, TelemetryAware { +public class OrderBy extends UnaryPlan + implements + PostAnalysisVerificationAware, + PostOptimizationVerificationAware, + TelemetryAware, + SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "OrderBy", OrderBy::new); private final List order; @@ -109,4 +115,9 @@ public void postAnalysisVerification(Failures failures) { } }); } + + @Override + public void postOptimizationVerification(Failures failures) { + failures.add(fail(this, "Unbounded sort not supported yet [{}] please add a limit", this.sourceText())); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java index e12a8cb557fde..a36341f60525a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java @@ -25,7 +25,7 @@ /** * A {@code Project} is a {@code Plan} with one child. In {@code SELECT x FROM y}, the "SELECT" statement is a Project. */ -public class Project extends UnaryPlan { +public class Project extends UnaryPlan implements SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Project", Project::new); private final List projections; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java index d691507b62cb3..f111b5d03edb3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java @@ -24,7 +24,7 @@ import static org.elasticsearch.xpack.esql.common.Failure.fail; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; -public abstract class RegexExtract extends UnaryPlan implements GeneratingPlan, PostAnalysisVerificationAware { +public abstract class RegexExtract extends UnaryPlan implements GeneratingPlan, PostAnalysisVerificationAware, SortAgnostic { protected final Expression input; protected final List extractedFields; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java index 7887d8ed66b99..c609bfdae87e7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Objects; -public class Rename extends UnaryPlan implements TelemetryAware { +public class Rename extends UnaryPlan implements TelemetryAware, SortAgnostic { private final List renamings; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/SortAgnostic.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/SortAgnostic.java new file mode 100644 index 0000000000000..3955b542ca496 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/SortAgnostic.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.logical; + +/** + * This interface is intended to check redundancy of a previous SORT. + *

+ * + * An example is with commands that compute values record by record, regardless of the input order + * and that don't rely on the context (intended as previous/next records). + * + *

+ *

+ * + * Example 1: if a MY_COMMAND that implements this interface is used between two sorts, + * then we can assume that + *

+ * + * | SORT x, y, z | MY_COMMAND | SORT a, b, c + * + *

+ * is equivalent to + *

+ * + * | MY_COMMAND | SORT a, b, c + * + * + *

+ *

+ * + * Example 2: commands that make previous order irrelevant, eg. because they collapse the results; + * STATS is one of them, eg. + * + *

+ * + * | SORT x, y, z | STATS count(*) + * + *

+ * is equivalent to + *

+ * + * | STATS count(*) + * + *

+ * + * and if MY_COMMAND implements this interface, then + * + *

+ * + * | SORT x, y, z | MY_COMMAND | STATS count(*) + * + *

+ * is equivalent to + *

+ * + * | MY_COMMAND | STATS count(*) + * + * + *

+ *

+ * + * In all the other cases, eg. if the command does not implement this interface + * then we assume that the previous SORT is still relevant and cannot be pruned. + * + *

+ *

+ * + * Eg. LIMIT does not implement this interface, because + * + *

+ * + * | SORT x, y, z | LIMIT 10 | SORT a, b, c + * + *

+ * is NOT equivalent to + *

+ * + * | LIMIT 10 | SORT a, b, c + * + * + *

+ *

+ * + * For n-ary plans that implement this interface, + * we assume that the above applies to all the children + * + */ +public interface SortAgnostic {} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnaryPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnaryPlan.java index ea9a760ef5dc4..6160c82c78c0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnaryPlan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnaryPlan.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.plan.logical; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.tree.Source; import java.util.Collections; @@ -20,6 +21,7 @@ public abstract class UnaryPlan extends LogicalPlan { private final LogicalPlan child; + private AttributeSet lazyOutputSet; protected UnaryPlan(Source source, LogicalPlan child) { super(source, Collections.singletonList(child)); @@ -42,6 +44,14 @@ public List output() { return child.output(); } + public AttributeSet outputSet() { + if (lazyOutputSet == null) { + List output = output(); + lazyOutputSet = (output == child.output() ? child.outputSet() : new AttributeSet(output)); + } + return lazyOutputSet; + } + @Override public int hashCode() { return Objects.hashCode(child()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 997bff70663bd..14877abb62272 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.logical.BinaryPlan; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.SortAgnostic; import java.io.IOException; import java.util.ArrayList; @@ -32,7 +33,7 @@ import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; import static org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.LEFT; -public class Join extends BinaryPlan implements PostAnalysisVerificationAware { +public class Join extends BinaryPlan implements PostAnalysisVerificationAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Join", Join::new); private final JoinConfig config; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java index 3c2d49567813c..6d3648d8e37b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java @@ -136,6 +136,7 @@ public Integer estimatedRowSize() { public PhysicalPlan estimateRowSize(State state) { state.add(false, aggregates); // The groupings are contained within the aggregates int size = state.consumeAllFields(true); + size = Math.max(size, 1); return Objects.equals(this.estimatedRowSize, size) ? this : new AggregateExec(source(), child(), groupings, aggregates, mode, intermediateAttributes, size); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java index a3fc62d935795..60e7eb535f444 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.NodeUtils; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -204,6 +205,15 @@ public static boolean isSourceAttribute(Attribute attr) { return DOC_ID_FIELD.getName().equals(attr.name()); } + public boolean hasScoring() { + for (Attribute a : attrs()) { + if (a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE)) { + return true; + } + } + return false; + } + @Override protected NodeInfo info() { return NodeInfo.create( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java index 96214652b87cb..5519e7fbc7083 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java @@ -37,7 +37,6 @@ public enum StatsType { } public record Stat(String name, StatsType type, QueryBuilder query) { - public QueryBuilder filter(QueryBuilder sourceQuery) { return query == null ? sourceQuery : Queries.combine(Queries.Clause.FILTER, asList(sourceQuery, query)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FilterExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FilterExec.java index 0802fc3423b23..26b05c98b79d8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FilterExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FilterExec.java @@ -9,14 +9,12 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.List; import java.util.Objects; public class FilterExec extends UnaryExec { @@ -63,11 +61,6 @@ public Expression condition() { return condition; } - @Override - public List output() { - return child().output(); - } - @Override public int hashCode() { return Objects.hash(condition, child()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java deleted file mode 100644 index 9d53e828f4f81..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.plan.physical; - -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.Order; -import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; - -public class OrderExec extends UnaryExec { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( - PhysicalPlan.class, - "OrderExec", - OrderExec::new - ); - - private final List order; - - public OrderExec(Source source, PhysicalPlan child, List order) { - super(source, child); - this.order = order; - } - - private OrderExec(StreamInput in) throws IOException { - this( - Source.readFrom((PlanStreamInput) in), - in.readNamedWriteable(PhysicalPlan.class), - in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new) - ); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - Source.EMPTY.writeTo(out); - out.writeNamedWriteable(child()); - out.writeCollection(order()); - } - - @Override - public String getWriteableName() { - return ENTRY.name; - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, OrderExec::new, child(), order); - } - - @Override - public OrderExec replaceChild(PhysicalPlan newChild) { - return new OrderExec(source(), newChild, order); - } - - public List order() { - return order; - } - - @Override - public int hashCode() { - return Objects.hash(order, child()); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - OrderExec other = (OrderExec) obj; - - return Objects.equals(order, other.order) && Objects.equals(child(), other.child()); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java index bbf0c681bec7d..c88e77e4972ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java @@ -101,6 +101,7 @@ public PhysicalPlan estimateRowSize(State state) { final boolean needsSortedDocIds = output.stream().anyMatch(a -> a.dataType() == DataType.DOC_DATA_TYPE); state.add(needsSortedDocIds, output); int size = state.consumeAllFields(true); + size = Math.max(size, 1); return Objects.equals(this.estimatedRowSize, size) ? this : new TopNExec(source(), child(), order, limit, size); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/UnaryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/UnaryExec.java index 7125a4eeeb55b..d787faf7b1b0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/UnaryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/UnaryExec.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.plan.physical; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.tree.Source; import java.util.Collections; @@ -17,6 +18,7 @@ public abstract class UnaryExec extends PhysicalPlan { private final PhysicalPlan child; + private AttributeSet lazyOutputSet; protected UnaryExec(Source source, PhysicalPlan child) { super(source, Collections.singletonList(child)); @@ -39,6 +41,16 @@ public List output() { return child.output(); } + @Override + public AttributeSet outputSet() { + if (lazyOutputSet == null) { + List output = output(); + lazyOutputSet = (output == child.output() ? child.outputSet() : new AttributeSet(output)); + return lazyOutputSet; + } + return lazyOutputSet; + } + @Override public int hashCode() { return Objects.hashCode(child()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 346eea7ad7dfb..82db9bc61b045 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -180,10 +180,8 @@ private static Stream, Tuple>> typeAndNames(Class types = List.of("GeoPoint", "CartesianPoint"); extraConfigs = SPATIAL_EXTRA_CONFIGS; } else if (clazz == SpatialExtent.class) { - return Stream.concat( - combine(clazz, List.of("GeoPoint", "CartesianPoint"), SPATIAL_EXTRA_CONFIGS), - combine(clazz, List.of("GeoShape", "CartesianShape"), List.of("")) - ); + types = List.of("GeoPoint", "CartesianPoint", "GeoShape", "CartesianShape"); + extraConfigs = SPATIAL_EXTRA_CONFIGS; } else if (Values.class.isAssignableFrom(clazz)) { // TODO can't we figure this out from the function itself? types = List.of("Int", "Long", "Double", "Boolean", "BytesRef"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 46a3bef5fe28b..e56d9ef177138 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -38,6 +38,8 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.search.NestedHelper; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; @@ -47,7 +49,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; -import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; @@ -72,6 +73,8 @@ import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; public class EsPhysicalOperationProviders extends AbstractPhysicalOperationProviders { + private static final Logger logger = LogManager.getLogger(EsPhysicalOperationProviders.class); + /** * Context of each shard we're operating against. */ @@ -157,14 +160,13 @@ public Function querySuppl @Override public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, LocalExecutionPlannerContext context) { final LuceneOperator.Factory luceneFactory; + logger.trace("Query Exec is {}", esQueryExec); List sorts = esQueryExec.sorts(); assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized"; int rowEstimatedSize = esQueryExec.estimatedRowSize(); int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold(context.foldCtx()) : NO_LIMIT; - boolean scoring = esQueryExec.attrs() - .stream() - .anyMatch(a -> a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE)); + boolean scoring = esQueryExec.hasScoring(); if ((sorts != null && sorts.isEmpty() == false)) { List> sortBuilders = new ArrayList<>(sorts.size()); for (Sort sort : sorts) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java index dafba5e92322c..b0e10ca7975ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java @@ -109,12 +109,11 @@ public Layout build() { for (ChannelSet set : channels) { int channel = numberOfChannels++; for (NameId id : set.nameIds) { + // Duplicate name ids would mean that have 2 channels that are declared under the same id. That makes no sense - which + // channel should subsequent operators use, then, when they want to refer to this id? + assert (layout.containsKey(id) == false) : "Duplicate name ids are not allowed in layouts"; ChannelAndType next = new ChannelAndType(channel, set.type); - ChannelAndType prev = layout.put(id, next); - // Do allow multiple name to point to the same channel - see https://github.com/elastic/elasticsearch/pull/100238 - // if (prev != null) { - // throw new IllegalArgumentException("Name [" + id + "] is on two channels [" + prev + "] and [" + next + "]"); - // } + layout.put(id, next); } } return new DefaultLayout(Collections.unmodifiableMap(layout), numberOfChannels); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index a8efd3b49ccf9..00adfd67dfba0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; @@ -21,7 +22,6 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; -import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -33,7 +33,6 @@ import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Filter; -import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsSourceExec; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; @@ -41,7 +40,6 @@ import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeSourceExec; import org.elasticsearch.xpack.esql.plan.physical.FragmentExec; -import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.mapper.LocalMapper; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; @@ -50,15 +48,14 @@ import org.elasticsearch.xpack.esql.stats.SearchStats; import java.util.ArrayList; -import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; import static java.util.Arrays.asList; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; +import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER; import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource; @@ -109,7 +106,7 @@ public static Set planConcreteIndices(PhysicalPlan plan) { return Set.of(); } var indices = new LinkedHashSet(); - forEachFromRelation(plan, relation -> indices.addAll(relation.concreteIndices())); + forEachRelation(plan, relation -> indices.addAll(relation.concreteIndices())); return indices; } @@ -121,42 +118,16 @@ public static String[] planOriginalIndices(PhysicalPlan plan) { return Strings.EMPTY_ARRAY; } var indices = new LinkedHashSet(); - forEachFromRelation(plan, relation -> indices.addAll(asList(Strings.commaDelimitedListToStringArray(relation.indexPattern())))); + forEachRelation(plan, relation -> indices.addAll(asList(Strings.commaDelimitedListToStringArray(relation.indexPattern())))); return indices.toArray(String[]::new); } - /** - * Iterates over the plan and applies the action to each {@link EsRelation} node. - *

- * This method ignores the right side of joins. - *

- */ - private static void forEachFromRelation(PhysicalPlan plan, Consumer action) { - // Take the non-join-side fragments - forEachUpWithChildren(plan, FragmentExec.class, fragment -> { - // Take the non-join-side relations - forEachUpWithChildren( - fragment.fragment(), - EsRelation.class, - action, - node -> node instanceof Join join ? List.of(join.left()) : node.children() - ); - }, node -> node instanceof LookupJoinExec join ? List.of(join.left()) : node.children()); - } - - /** - * Similar to {@link Node#forEachUp(Class, Consumer)}, but with a custom callback to get the node children. - */ - private static , E extends T> void forEachUpWithChildren( - T node, - Class typeToken, - Consumer action, - Function> childrenGetter - ) { - childrenGetter.apply(node).forEach(c -> forEachUpWithChildren(c, typeToken, action, childrenGetter)); - if (typeToken.isInstance(node)) { - action.accept(typeToken.cast(node)); - } + private static void forEachRelation(PhysicalPlan plan, Consumer action) { + plan.forEachDown(FragmentExec.class, f -> f.fragment().forEachDown(EsRelation.class, r -> { + if (r.indexMode() != IndexMode.LOOKUP) { + action.accept(r); + } + })); } public static PhysicalPlan localPlan( @@ -290,7 +261,7 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field case DOC_DATA_TYPE -> ElementType.DOC; case TSID_DATA_TYPE -> ElementType.BYTES_REF; case GEO_POINT, CARTESIAN_POINT -> fieldExtractPreference == DOC_VALUES ? ElementType.LONG : ElementType.BYTES_REF; - case GEO_SHAPE, CARTESIAN_SHAPE -> ElementType.BYTES_REF; + case GEO_SHAPE, CARTESIAN_SHAPE -> fieldExtractPreference == EXTRACT_SPATIAL_BOUNDS ? ElementType.INT : ElementType.BYTES_REF; case PARTIAL_AGG, AGGREGATE_METRIC_DOUBLE -> ElementType.COMPOSITE; case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT -> throw EsqlIllegalArgumentException .illegalDataType(dataType); @@ -306,11 +277,4 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field new NoopCircuitBreaker("noop-esql-breaker"), BigArrays.NON_RECYCLING_INSTANCE ); - - /** - * Returns DOC_VALUES if the given boolean is set. - */ - public static MappedFieldType.FieldExtractPreference extractPreference(boolean hasPreference) { - return hasPreference ? DOC_VALUES : NONE; - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/LocalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/LocalMapper.java index f95ae0e0783e5..217737de5309b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/LocalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/LocalMapper.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.esql.plan.logical.LeafPlan; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.join.Join; @@ -28,7 +27,6 @@ import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec; -import org.elasticsearch.xpack.esql.plan.physical.OrderExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; @@ -81,10 +79,6 @@ private PhysicalPlan mapUnary(UnaryPlan unary) { return new LimitExec(limit.source(), mappedChild, limit.limit()); } - if (unary instanceof OrderBy o) { - return new OrderExec(o.source(), mappedChild, o.order()); - } - if (unary instanceof TopN topN) { return new TopNExec(topN.source(), mappedChild, topN.order(), topN.limit(), null); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java index 8a4325ed84b2a..8ea19f545e67b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/Mapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec; -import org.elasticsearch.xpack.esql.plan.physical.OrderExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; import org.elasticsearch.xpack.esql.plan.physical.UnaryExec; @@ -105,7 +104,7 @@ private PhysicalPlan mapUnary(UnaryPlan unary) { return enrichExec.child(); } if (f instanceof UnaryExec unaryExec) { - if (f instanceof LimitExec || f instanceof ExchangeExec || f instanceof OrderExec || f instanceof TopNExec) { + if (f instanceof LimitExec || f instanceof ExchangeExec || f instanceof TopNExec) { return f; } else { return unaryExec.child(); @@ -161,11 +160,6 @@ private PhysicalPlan mapUnary(UnaryPlan unary) { return new LimitExec(limit.source(), mappedChild, limit.limit()); } - if (unary instanceof OrderBy o) { - mappedChild = addExchangeForFragment(o, mappedChild); - return new OrderExec(o.source(), mappedChild, o.order()); - } - if (unary instanceof TopN topN) { mappedChild = addExchangeForFragment(topN, mappedChild); return new TopNExec(topN.source(), mappedChild, topN.order(), topN.limit(), null); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java index 19ed77405daa2..d0b511a4ee846 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java @@ -197,8 +197,7 @@ void runComputeOnRemoteCluster( }))) { var exchangeSource = new ExchangeSourceHandler( configuration.pragmas().exchangeBufferSize(), - transportService.getThreadPool().executor(ThreadPool.Names.SEARCH), - computeListener.acquireAvoid() + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH) ); try (Releasable ignored = exchangeSource.addEmptySink()) { exchangeSink.addCompletionListener(computeListener.acquireAvoid()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java index 3d358b8c7a8a2..c8b8e84fd2478 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java @@ -47,7 +47,8 @@ final class ComputeListener implements Releasable { * Acquires a new listener that doesn't collect result */ ActionListener acquireAvoid() { - return refs.acquire().delegateResponse((l, e) -> { + var listener = ActionListener.assertAtLeastOnce(refs.acquire()); + return listener.delegateResponse((l, e) -> { try { runOnFailure.run(); } finally { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 71c2a65037e9a..1e6f003dd9d77 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -190,16 +190,16 @@ public void execute( * entire plan. */ List outputAttributes = physicalPlan.output(); + var exchangeSource = new ExchangeSourceHandler( + queryPragmas.exchangeBufferSize(), + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH) + ); + listener = ActionListener.runBefore(listener, () -> exchangeService.removeExchangeSourceHandler(sessionId)); + exchangeService.addExchangeSourceHandler(sessionId, exchangeSource); try (var computeListener = new ComputeListener(transportService.getThreadPool(), cancelQueryOnFailure, listener.map(profiles -> { execInfo.markEndQuery(); // TODO: revisit this time recording model as part of INLINESTATS improvements return new Result(outputAttributes, collectedPages, profiles, execInfo); }))) { - var exchangeSource = new ExchangeSourceHandler( - queryPragmas.exchangeBufferSize(), - transportService.getThreadPool().executor(ThreadPool.Names.SEARCH), - ActionListener.runBefore(computeListener.acquireAvoid(), () -> exchangeService.removeExchangeSourceHandler(sessionId)) - ); - exchangeService.addExchangeSourceHandler(sessionId, exchangeSource); try (Releasable ignored = exchangeSource.addEmptySink()) { // run compute on the coordinator final AtomicBoolean localClusterWasInterrupted = new AtomicBoolean(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java index 7020932819421..d228aa638a13a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java @@ -428,7 +428,7 @@ private void runComputeOnDataNode( task.addListener( () -> exchangeService.finishSinkHandler(externalId, new TaskCancelledException(task.getReasonCancelled())) ); - var exchangeSource = new ExchangeSourceHandler(1, esqlExecutor, computeListener.acquireAvoid()); + var exchangeSource = new ExchangeSourceHandler(1, esqlExecutor); exchangeSource.addRemoteSink(internalSink::fetchPageAsync, true, () -> {}, 1, ActionListener.noop()); var reductionListener = computeListener.acquireCompute(); computeService.runCompute( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java index a4007a520ed30..f5f51029ae8a6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java @@ -7,7 +7,8 @@ package org.elasticsearch.xpack.esql.plugin; -import org.elasticsearch.ResourceNotFoundException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; @@ -16,10 +17,8 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.operator.exchange.ExchangeService; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -32,12 +31,11 @@ import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.esql.action.EsqlAsyncStopAction; +import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.action.EsqlQueryTask; import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; @@ -55,6 +53,8 @@ public class TransportEsqlAsyncStopAction extends HandledTransportAction listener) { String asyncIdStr = asyncId.getEncoded(); - TransportEsqlQueryAction.EsqlQueryListener asyncListener = queryAction.getAsyncListener(asyncIdStr); - if (asyncListener == null) { + EsqlQueryTask asyncTask = getEsqlQueryTask(asyncId); + GetAsyncResultRequest getAsyncResultRequest = new GetAsyncResultRequest(asyncIdStr); + if (asyncTask == null) { // This should mean one of the two things: either bad request ID, or the query has already finished // In both cases, let regular async get deal with it. - var getAsyncResultRequest = new GetAsyncResultRequest(asyncIdStr); - // TODO: this should not be happening, but if the listener is not registered and the query is not finished, - // we give it some time to finish - getAsyncResultRequest.setWaitForCompletionTimeout(new TimeValue(1, TimeUnit.SECONDS)); + logger.debug("Async stop for task {}, no task present - passing to GetAsyncResultRequest", asyncIdStr); getResultsAction.execute(task, getAsyncResultRequest, listener); return; } - try { - EsqlQueryTask asyncTask = AsyncTaskIndexService.getTask(taskManager, asyncId, EsqlQueryTask.class); - if (false == security.currentUserHasAccessToTask(asyncTask)) { - throw new ResourceNotFoundException(asyncId + " not found"); + logger.debug("Async stop for task {} - stopping", asyncIdStr); + final EsqlExecutionInfo esqlExecutionInfo = asyncTask.executionInfo(); + if (esqlExecutionInfo != null) { + esqlExecutionInfo.markAsPartial(); + } + Runnable getResults = () -> getResultsAction.execute(task, getAsyncResultRequest, listener); + exchangeService.finishSessionEarly(sessionID(asyncId), ActionListener.running(() -> { + if (asyncTask.addCompletionListener(() -> ActionListener.running(getResults)) == false) { + getResults.run(); } + })); + } + + private EsqlQueryTask getEsqlQueryTask(AsyncExecutionId asyncId) { + try { + return AsyncTaskIndexService.getTaskAndCheckAuthentication(taskManager, security, asyncId, EsqlQueryTask.class); } catch (IOException e) { - throw new ResourceNotFoundException(asyncId + " not found", e); - } - // Here we will wait for both the response to become available and for the finish operation to complete - var responseHolder = new AtomicReference(); - try (var refs = new EsqlRefCountingListener(listener.map(unused -> responseHolder.get()))) { - asyncListener.addListener(refs.acquire().map(r -> { - responseHolder.set(r); - return null; - })); - asyncListener.markAsPartial(); - exchangeService.finishSessionEarly(sessionID(asyncId), refs.acquire()); + return null; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index a32b4591943f4..d83239545c383 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -13,14 +13,12 @@ import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.operator.exchange.ExchangeService; @@ -83,8 +81,6 @@ public class TransportEsqlQueryAction extends HandledTransportAction asyncListeners = ConcurrentCollections.newConcurrentMap(); @Inject @SuppressWarnings("this-escape") @@ -183,41 +179,11 @@ private void doExecuteForked(Task task, EsqlQueryRequest request, ActionListener } } - // Subscribable listener that can keep track of the EsqlExecutionInfo - // Used to mark an async query as partial if it is stopped - public static class EsqlQueryListener extends SubscribableListener { - private EsqlExecutionInfo executionInfo; - - public EsqlQueryListener(EsqlExecutionInfo executionInfo) { - this.executionInfo = executionInfo; - } - - public EsqlExecutionInfo getExecutionInfo() { - return executionInfo; - } - - public void markAsPartial() { - if (executionInfo != null) { - executionInfo.markAsPartial(); - } - } - } - @Override public void execute(EsqlQueryRequest request, EsqlQueryTask task, ActionListener listener) { // set EsqlExecutionInfo on async-search task so that it is accessible to GET _query/async while the query is still running task.setExecutionInfo(createEsqlExecutionInfo(request)); - // Since the request is async here, we need to wrap the listener in a SubscribableListener so that we can collect the results from - // other endpoints, such as _query/async/stop - EsqlQueryListener subListener = new EsqlQueryListener(task.executionInfo()); - String asyncExecutionId = task.getExecutionId().getEncoded(); - subListener.addListener(ActionListener.runAfter(listener, () -> asyncListeners.remove(asyncExecutionId))); - asyncListeners.put(asyncExecutionId, subListener); - ActionListener.run(subListener, l -> innerExecute(task, request, l)); - } - - public EsqlQueryListener getAsyncListener(String executionId) { - return asyncListeners.get(executionId); + ActionListener.run(listener, l -> innerExecute(task, request, l)); } private void innerExecute(Task task, EsqlQueryRequest request, ActionListener listener) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java index 7db81069f9d3c..10f43b66ebdf7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.search.SearchService; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -98,9 +97,9 @@ private Set fullTextFunctions(LogicalPlan plan) { } public Set indexNames(LogicalPlan plan) { - Holder> indexNames = new Holder<>(); - plan.forEachDown(EsRelation.class, esRelation -> indexNames.set(esRelation.concreteIndices())); - return indexNames.get(); + Set indexNames = new HashSet<>(); + plan.forEachDown(EsRelation.class, esRelation -> indexNames.addAll(esRelation.concreteIndices())); + return indexNames; } public LogicalPlan planWithResolvedQueryBuilders(LogicalPlan plan, Map newQueryBuilders) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index b5509883aedc4..986f1594a4b3f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -557,6 +557,16 @@ public static long dateNanosToLong(String dateNano, DateFormatter formatter) { return DateUtils.toLong(parsed); } + public static String dateWithTypeToString(long dateTime, DataType type) { + if (type == DATETIME) { + return dateTimeToString(dateTime); + } + if (type == DATE_NANOS) { + return nanoTimeToString(dateTime); + } + throw new IllegalArgumentException("Unsupported data type [" + type + "]"); + } + public static String dateTimeToString(long dateTime) { return DEFAULT_DATE_TIME_FORMATTER.formatMillis(dateTime); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index dd3f923eb3939..2596b7eff8bb5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -609,7 +609,7 @@ void executeSubPlan( bigArrays, ByteSizeValue.ofBytes(randomLongBetween(1, BlockFactory.DEFAULT_MAX_BLOCK_PRIMITIVE_ARRAY_SIZE.getBytes() * 2)) ); - ExchangeSourceHandler exchangeSource = new ExchangeSourceHandler(between(1, 64), executor, ActionListener.noop()); + ExchangeSourceHandler exchangeSource = new ExchangeSourceHandler(between(1, 64), executor); ExchangeSinkHandler exchangeSink = new ExchangeSinkHandler(blockFactory, between(1, 64), threadPool::relativeTimeInMillis); LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java index dcb83dadfcf96..698291a54fa68 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java @@ -144,10 +144,6 @@ public void testNamedParams() throws IOException { } public void testNamedParamsForIdentifiersPatterns() throws IOException { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); String query = randomAlphaOfLengthBetween(1, 100); boolean columnar = randomBoolean(); Locale locale = randomLocale(random()); @@ -260,10 +256,6 @@ public void testInvalidParams() throws IOException { } public void testInvalidParamsForIdentifiersPatterns() throws IOException { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); String query = randomAlphaOfLengthBetween(1, 100); boolean columnar = randomBoolean(); Locale locale = randomLocale(random()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 2940ca0f53658..87929fca192b4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2325,13 +2325,37 @@ public void testRateRequiresCounterTypes() { ); } - public void testCoalesceWithMixedNumericTypes() { + public void testConditionalFunctionsWithMixedNumericTypes() { LogicalPlan plan = analyze(""" from test | eval x = coalesce(salary_change, null, 0), y = coalesce(languages, null, 0), z = coalesce(languages.long, null, 0) , w = coalesce(salary_change, null, 0::long) | keep x, y, z, w """, "mapping-default.json"); + validateConditionalFunctions(plan); + + plan = analyze(""" + from test + | eval x = case(languages == 1, salary_change, languages == 2, salary, languages == 3, salary_change.long, 0) + , y = case(languages == 1, salary_change.int, languages == 2, salary, 0) + , z = case(languages == 1, salary_change.long, languages == 2, salary, 0::long) + , w = case(languages == 1, salary_change, languages == 2, salary, languages == 3, salary_change.long, null) + | keep x, y, z, w + """, "mapping-default.json"); + validateConditionalFunctions(plan); + + plan = analyze(""" + from test + | eval x = greatest(salary_change, salary, salary_change.long) + , y = least(salary_change.int, salary) + , z = greatest(salary_change.long, salary, null) + , w = least(null, salary_change, salary_change.long, salary, null) + | keep x, y, z, w + """, "mapping-default.json"); + validateConditionalFunctions(plan); + } + + private void validateConditionalFunctions(LogicalPlan plan) { var limit = as(plan, Limit.class); var esqlProject = as(limit.child(), EsqlProject.class); List projections = esqlProject.projections(); @@ -2351,10 +2375,6 @@ public void testCoalesceWithMixedNumericTypes() { } public void testNamedParamsForIdentifiers() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); assertProjectionWithMapping( """ from test @@ -2445,10 +2465,6 @@ public void testNamedParamsForIdentifiers() { } public void testInvalidNamedParamsForIdentifiers() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // missing field assertError( """ @@ -2518,10 +2534,6 @@ public void testInvalidNamedParamsForIdentifiers() { } public void testNamedParamsForIdentifierPatterns() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); assertProjectionWithMapping( """ from test @@ -2552,10 +2564,6 @@ public void testNamedParamsForIdentifierPatterns() { } public void testInvalidNamedParamsForIdentifierPatterns() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // missing pattern assertError( """ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 09cfefc95ed88..f33178ea2d39d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1611,108 +1611,162 @@ public void testTermTargetsExistingField() throws Exception { assertEquals("1:38: Unknown column [first_name]", error("from test | keep emp_no | where term(first_name, \"Anna\")")); } - public void testCoalesceWithMixedNumericTypes() { - assertEquals( - "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(languages, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages.long, height)] must be [long], found value [height] type [double]", - error("from test | eval x = coalesce(languages.long, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(salary, languages.long)] must be [integer], found value [languages.long] type [long]", - error("from test | eval x = coalesce(salary, languages.long)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages.short, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(languages.short, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages.byte, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(languages.byte, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages, height.float)] must be [integer], found value [height.float] type [double]", - error("from test | eval x = coalesce(languages, height.float)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages, height.scaled_float)] must be [integer], " - + "found value [height.scaled_float] type [double]", - error("from test | eval x = coalesce(languages, height.scaled_float)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages, height.half_float)] must be [integer], " - + "found value [height.half_float] type [double]", - error("from test | eval x = coalesce(languages, height.half_float)") - ); + public void testConditionalFunctionsWithMixedNumericTypes() { + for (String functionName : List.of("coalesce", "greatest", "least")) { + assertEquals( + "1:22: second argument of [" + functionName + "(languages, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages.long, height)] must be [long], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages.long, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(salary, languages.long)] must be [integer], found value [languages.long] type [long]", + error("from test | eval x = " + functionName + "(salary, languages.long)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages.short, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages.short, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages.byte, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages.byte, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages, height.float)] must be [integer], found value [height.float] type [double]", + error("from test | eval x = " + functionName + "(languages, height.float)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages, height.scaled_float)] must be [integer], " + + "found value [height.scaled_float] type [double]", + error("from test | eval x = " + functionName + "(languages, height.scaled_float)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages, height.half_float)] must be [integer], " + + "found value [height.half_float] type [double]", + error("from test | eval x = " + functionName + "(languages, height.half_float)") + ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages.long, height)] must be [long], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages.long, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, salary, languages.long)] must be [integer], " - + "found value [languages.long] type [long]", - error("from test | eval x = coalesce(null, salary, languages.long)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages.short, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages.short, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages.byte, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages.byte, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height.float)] must be [integer], " - + "found value [height.float] type [double]", - error("from test | eval x = coalesce(null, languages, height.float)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height.scaled_float)] must be [integer], " - + "found value [height.scaled_float] type [double]", - error("from test | eval x = coalesce(null, languages, height.scaled_float)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height.half_float)] must be [integer], " - + "found value [height.half_float] type [double]", - error("from test | eval x = coalesce(null, languages, height.half_float)") - ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages.long, height)] must be [long], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages.long, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, salary, languages.long)] must be [integer], " + + "found value [languages.long] type [long]", + error("from test | eval x = " + functionName + "(null, salary, languages.long)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages.short, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages.short, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages.byte, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages.byte, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height.float)] must be [integer], " + + "found value [height.float] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height.float)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height.scaled_float)] must be [integer], " + + "found value [height.scaled_float] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height.scaled_float)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height.half_float)] must be [integer], " + + "found value [height.half_float] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height.half_float)") + ); - // counter - assertEquals( - "1:23: second argument of [coalesce(network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", - error("FROM tests | eval x = coalesce(network.bytes_in, 0)", tsdb) - ); + // counter + assertEquals( + "1:23: second argument of [" + + functionName + + "(network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", + error("FROM tests | eval x = " + functionName + "(network.bytes_in, 0)", tsdb) + ); - assertEquals( - "1:23: second argument of [coalesce(network.bytes_in, to_long(0))] must be [counter_long], " - + "found value [to_long(0)] type [long]", - error("FROM tests | eval x = coalesce(network.bytes_in, to_long(0))", tsdb) - ); - assertEquals( - "1:23: second argument of [coalesce(network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", - error("FROM tests | eval x = coalesce(network.bytes_in, 0.0)", tsdb) - ); + assertEquals( + "1:23: second argument of [" + + functionName + + "(network.bytes_in, to_long(0))] must be [counter_long], " + + "found value [to_long(0)] type [long]", + error("FROM tests | eval x = " + functionName + "(network.bytes_in, to_long(0))", tsdb) + ); + assertEquals( + "1:23: second argument of [" + + functionName + + "(network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", + error("FROM tests | eval x = " + functionName + "(network.bytes_in, 0.0)", tsdb) + ); - assertEquals( - "1:23: third argument of [coalesce(null, network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", - error("FROM tests | eval x = coalesce(null, network.bytes_in, 0)", tsdb) - ); + assertEquals( + "1:23: third argument of [" + + functionName + + "(null, network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", + error("FROM tests | eval x = " + functionName + "(null, network.bytes_in, 0)", tsdb) + ); + + assertEquals( + "1:23: third argument of [" + + functionName + + "(null, network.bytes_in, to_long(0))] must be [counter_long], " + + "found value [to_long(0)] type [long]", + error("FROM tests | eval x = " + functionName + "(null, network.bytes_in, to_long(0))", tsdb) + ); + assertEquals( + "1:23: third argument of [" + + functionName + + "(null, network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", + error("FROM tests | eval x = " + functionName + "(null, network.bytes_in, 0.0)", tsdb) + ); + } + // case, a subset tests of coalesce/greatest/least assertEquals( - "1:23: third argument of [coalesce(null, network.bytes_in, to_long(0))] must be [counter_long], " - + "found value [to_long(0)] type [long]", - error("FROM tests | eval x = coalesce(null, network.bytes_in, to_long(0))", tsdb) + "1:22: third argument of [case(languages == 1, salary, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = case(languages == 1, salary, height)") ); assertEquals( - "1:23: third argument of [coalesce(null, network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", - error("FROM tests | eval x = coalesce(null, network.bytes_in, 0.0)", tsdb) + "1:23: third argument of [case(name == \"a\", network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", + error("FROM tests | eval x = case(name == \"a\", network.bytes_in, 0)", tsdb) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 61138b179caf2..c03ff0cfbeeee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -953,7 +953,7 @@ protected static void renderDocs(String name) throws IOException { "// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.\n\n"; private static final String PREVIEW_CALLOUT = - "\npreview::[\"Do not use `VALUES` on production environments. This functionality is in technical preview and " + "\npreview::[\"Do not use on production environments. This functionality is in technical preview and " + "may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview " + "are not subject to the support SLA of official GA features.\"]\n"; @@ -1253,7 +1253,7 @@ private static void renderKibanaFunctionDefinition( builder.startObject(); builder.field("name", arg.name()); if (arg.mapArg()) { - builder.field("type", "function named parameters"); + builder.field("type", "function_named_parameters"); builder.field( "mapParams", arg.mapParams() diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java index 98c6e0ea8adc7..0e34e73d73cb1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java @@ -42,7 +42,7 @@ public static Iterable parameters() { read, TestCaseSupplier.dateCases(), DataType.DATETIME, - v -> ((Instant) v).toEpochMilli(), + v -> DateUtils.toLongMillis((Instant) v), emptyList() ); TestCaseSupplier.unary( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index c2d7565c3e6f4..f4d82f571dc88 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -43,7 +43,14 @@ public static Iterable parameters() { TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataType.LONG, b -> b ? 1L : 0L, List.of()); // datetimes - TestCaseSupplier.unary(suppliers, read, TestCaseSupplier.dateCases(), DataType.LONG, v -> ((Instant) v).toEpochMilli(), List.of()); + TestCaseSupplier.unary( + suppliers, + read, + TestCaseSupplier.dateCases(), + DataType.LONG, + v -> DateUtils.toLongMillis((Instant) v), + List.of() + ); TestCaseSupplier.unary( suppliers, read, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java index c89f8c34b8456..e5a80de5b5eb2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -87,7 +87,7 @@ public static Iterable parameters() { "ToStringFromDatetimeEvaluator[field=" + read + "]", TestCaseSupplier.dateCases(), DataType.KEYWORD, - i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(((Instant) i).toEpochMilli())), + i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(DateUtils.toLongMillis((Instant) i))), List.of() ); TestCaseSupplier.unary( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index c41b1e14257ee..a5fe9d7c78b68 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -12,14 +12,21 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import org.hamcrest.Matcher; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; @@ -98,4 +105,38 @@ private static TestCaseSupplier.TestCase testCase( protected Expression build(Source source, List args) { return new EndsWith(source, args.get(0), args.get(1)); } + + public void testLuceneQuery_AllLiterals_NonTranslatable() { + var function = new EndsWith( + Source.EMPTY, + new Literal(Source.EMPTY, "test", DataType.KEYWORD), + new Literal(Source.EMPTY, "test", DataType.KEYWORD) + ); + + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + } + + public void testLuceneQuery_NonFoldableSuffix_NonTranslatable() { + var function = new EndsWith( + Source.EMPTY, + new FieldAttribute(Source.EMPTY, "field", new EsField("field", DataType.KEYWORD, Map.of(), true)), + new FieldAttribute(Source.EMPTY, "field", new EsField("suffix", DataType.KEYWORD, Map.of(), true)) + ); + + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + } + + public void testLuceneQuery_NonFoldableSuffix_Translatable() { + var function = new EndsWith( + Source.EMPTY, + new FieldAttribute(Source.EMPTY, "field", new EsField("suffix", DataType.KEYWORD, Map.of(), true)), + new Literal(Source.EMPTY, "a*b?c\\", DataType.KEYWORD) + ); + + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true)); + + var query = function.asQuery(TranslatorHandler.TRANSLATOR_HANDLER); + + assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "*a\\*b\\?c\\\\"))); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index 789059fb7b6ba..06d2757766060 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -12,13 +12,20 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; @@ -58,4 +65,38 @@ public static Iterable parameters() { protected Expression build(Source source, List args) { return new StartsWith(source, args.get(0), args.get(1)); } + + public void testLuceneQuery_AllLiterals_NonTranslatable() { + var function = new StartsWith( + Source.EMPTY, + new Literal(Source.EMPTY, "test", DataType.KEYWORD), + new Literal(Source.EMPTY, "test", DataType.KEYWORD) + ); + + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + } + + public void testLuceneQuery_NonFoldablePrefix_NonTranslatable() { + var function = new StartsWith( + Source.EMPTY, + new FieldAttribute(Source.EMPTY, "field", new EsField("field", DataType.KEYWORD, Map.of(), true)), + new FieldAttribute(Source.EMPTY, "field", new EsField("prefix", DataType.KEYWORD, Map.of(), true)) + ); + + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + } + + public void testLuceneQuery_NonFoldablePrefix_Translatable() { + var function = new StartsWith( + Source.EMPTY, + new FieldAttribute(Source.EMPTY, "field", new EsField("prefix", DataType.KEYWORD, Map.of(), true)), + new Literal(Source.EMPTY, "a*b?c\\", DataType.KEYWORD) + ); + + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true)); + + var query = function.asQuery(TranslatorHandler.TRANSLATOR_HANDLER); + + assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "a\\*b\\?c\\\\*"))); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 98f3d1d2d8d8e..6903e5dfce35d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -78,6 +77,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") @@ -141,7 +141,7 @@ public void testMissingFieldInFilterString() { /** * Expects - * Project[[last_name{r}#6]] + * Project[[last_name{f}#6]] * \_Eval[[null[KEYWORD] AS last_name]] * \_Limit[10000[INTEGER]] * \_EsRelation[test][_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen..] @@ -158,7 +158,7 @@ public void testMissingFieldInProject() { var project = as(localPlan, Project.class); var projections = project.projections(); assertThat(Expressions.names(projections), contains("last_name")); - as(projections.get(0), ReferenceAttribute.class); + as(projections.get(0), FieldAttribute.class); var eval = as(project.child(), Eval.class); assertThat(Expressions.names(eval.fields()), contains("last_name")); var alias = as(eval.fields().get(0), Alias.class); @@ -168,6 +168,7 @@ public void testMissingFieldInProject() { var limit = as(eval.child(), Limit.class); var source = as(limit.child(), EsRelation.class); + assertThat(Expressions.names(source.output()), not(contains("last_name"))); } /** @@ -192,6 +193,7 @@ public void testMissingFieldInSort() { var limit = as(project.child(), Limit.class); var source = as(limit.child(), EsRelation.class); + assertThat(Expressions.names(source.output()), not(contains("last_name"))); } /** @@ -199,8 +201,11 @@ public void testMissingFieldInSort() { * EsqlProject[[first_name{f}#7, last_name{r}#17]] * \_Limit[1000[INTEGER],true] * \_MvExpand[last_name{f}#10,last_name{r}#17] - * \_Limit[1000[INTEGER],false] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_Project[[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, gender{f}#8, hire_date{f}#13, job{f}#14, job.raw{f}#15, lang + * uages{f}#9, last_name{r}#10, long_noidx{f}#16, salary{f}#11]] + * \_Eval[[null[KEYWORD] AS last_name]] + * \_Limit[1000[INTEGER],false] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] */ public void testMissingFieldInMvExpand() { var plan = plan(""" @@ -212,14 +217,23 @@ public void testMissingFieldInMvExpand() { var testStats = statsForMissingField("last_name"); var localPlan = localPlan(plan, testStats); + // It'd be much better if this project was pushed down past the MvExpand, because MvExpand's cost scales with the number of + // involved attributes/columns. var project = as(localPlan, EsqlProject.class); var projections = project.projections(); assertThat(Expressions.names(projections), contains("first_name", "last_name")); var limit1 = asLimit(project.child(), 1000, true); var mvExpand = as(limit1.child(), MvExpand.class); - var limit2 = asLimit(mvExpand.child(), 1000, false); - as(limit2.child(), EsRelation.class); + var project2 = as(mvExpand.child(), Project.class); + var eval = as(project2.child(), Eval.class); + assertEquals(eval.fields().size(), 1); + var lastName = eval.fields().get(0); + assertEquals(lastName.name(), "last_name"); + assertEquals(lastName.child(), new Literal(EMPTY, null, DataType.KEYWORD)); + var limit2 = asLimit(eval.child(), 1000, false); + var relation = as(limit2.child(), EsRelation.class); + assertThat(Expressions.names(relation.output()), not(contains("last_name"))); } public static class MockFieldAttributeCommand extends UnaryPlan { @@ -275,6 +289,39 @@ public void testMissingFieldInNewCommand() { ), testStats ); + + var plan = plan(""" + from test + """); + var initialRelation = plan.collectLeaves().get(0); + FieldAttribute lastName = null; + for (Attribute attr : initialRelation.output()) { + if (attr.name().equals("last_name")) { + lastName = (FieldAttribute) attr; + } + } + + // Expects + // MockFieldAttributeCommand[last_name{f}#7] + // \_Project[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu + // ages{f}#6, last_name{r}#7, long_noidx{f}#13, salary{f}#8]] + // \_Eval[[null[KEYWORD] AS last_name]] + // \_Limit[1000[INTEGER],false] + // \_EsRelation[test][_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..] + LogicalPlan localPlan = localPlan(new MockFieldAttributeCommand(EMPTY, plan, lastName), testStats); + + var mockCommand = as(localPlan, MockFieldAttributeCommand.class); + var project = as(mockCommand.child(), Project.class); + var eval = as(project.child(), Eval.class); + var limit = asLimit(eval.child(), 1000); + var relation = as(limit.child(), EsRelation.class); + + assertThat(Expressions.names(eval.fields()), contains("last_name")); + var literal = as(eval.fields().get(0), Alias.class); + assertEquals(literal.child(), new Literal(EMPTY, null, DataType.KEYWORD)); + assertThat(Expressions.names(relation.output()), not(contains("last_name"))); + + assertEquals(Expressions.names(initialRelation.output()), Expressions.names(project.output())); } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 640f352bd110c..74ddcceb0505f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -37,8 +37,8 @@ import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; @@ -1282,9 +1282,9 @@ public void testMissingFieldsDoNotGetExtracted() { ) ); // emp_no - assertThat(projections.get(1), instanceOf(ReferenceAttribute.class)); + assertThat(projections.get(1), instanceOf(FieldAttribute.class)); // first_name - assertThat(projections.get(2), instanceOf(ReferenceAttribute.class)); + assertThat(projections.get(2), instanceOf(FieldAttribute.class)); // last_name --> first_name var nullAlias = Alias.unwrap(projections.get(8)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 9148914514930..bb7eb9cd230e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -96,6 +96,7 @@ import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.rules.logical.LiteralsOnTheRight; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneRedundantOrderBy; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownAndCombineLimits; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownEnrich; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownEval; @@ -1839,10 +1840,9 @@ public void testCombineOrderByThroughFilter() { /** * Expected - * TopN[[Order[first_name{f}#170,ASC,LAST]],1000[INTEGER]] - * \_MvExpand[first_name{f}#170] - * \_TopN[[Order[emp_no{f}#169,ASC,LAST]],1000[INTEGER]] - * \_EsRelation[test][avg_worked_seconds{f}#167, birth_date{f}#168, emp_n..] + * TopN[[Order[first_name{r}#5575,ASC,LAST]],1000[INTEGER]] + * \_MvExpand[first_name{f}#5565,first_name{r}#5575,null] + * \_EsRelation[test][_meta_field{f}#5570, emp_no{f}#5564, first_name{f}#..] */ public void testDontCombineOrderByThroughMvExpand() { LogicalPlan plan = optimizedPlan(""" @@ -1854,9 +1854,7 @@ public void testDontCombineOrderByThroughMvExpand() { var topN = as(plan, TopN.class); assertThat(orderNames(topN), contains("first_name")); var mvExpand = as(topN.child(), MvExpand.class); - topN = as(mvExpand.child(), TopN.class); - assertThat(orderNames(topN), contains("emp_no")); - as(topN.child(), EsRelation.class); + as(mvExpand.child(), EsRelation.class); } /** @@ -2065,12 +2063,10 @@ public void testMultipleLookupJoinWithSortAndLimit() { } /** - * Expected - * EsqlProject[[emp_no{f}#350, first_name{f}#351, salary{f}#352]] - * \_TopN[[Order[salary{f}#352,ASC,LAST], Order[first_name{f}#351,ASC,LAST]],5[INTEGER]] - * \_MvExpand[first_name{f}#351] - * \_TopN[[Order[emp_no{f}#350,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[employees][emp_no{f}#350, first_name{f}#351, salary{f}#352] + * EsqlProject[[emp_no{f}#10, first_name{r}#21, salary{f}#15]] + * \_TopN[[Order[salary{f}#15,ASC,LAST], Order[first_name{r}#21,ASC,LAST]],5[INTEGER]] + * \_MvExpand[first_name{f}#11,first_name{r}#21,null] + * \_EsRelation[test][_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, ..] */ public void testPushDownLimitThroughMultipleSort_AfterMvExpand() { LogicalPlan plan = optimizedPlan(""" @@ -2086,20 +2082,16 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand() { assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary", "first_name")); var mvExp = as(topN.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); - assertThat(orderNames(topN), contains("emp_no")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** * Expected - * EsqlProject[[emp_no{f}#361, first_name{f}#362, salary{f}#363]] - * \_TopN[[Order[first_name{f}#362,ASC,LAST]],5[INTEGER]] - * \_TopN[[Order[salary{f}#363,ASC,LAST]],5[INTEGER]] - * \_MvExpand[first_name{f}#362] - * \_TopN[[Order[emp_no{f}#361,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[employees][emp_no{f}#361, first_name{f}#362, salary{f}#363] + * EsqlProject[[emp_no{f}#2560, first_name{r}#2571, salary{f}#2565]] + * \_TopN[[Order[first_name{r}#2571,ASC,LAST]],5[INTEGER]] + * \_TopN[[Order[salary{f}#2565,ASC,LAST]],5[INTEGER]] + * \_MvExpand[first_name{f}#2561,first_name{r}#2571,null] + * \_EsRelation[test][_meta_field{f}#2566, emp_no{f}#2560, first_name{f}#..] */ public void testPushDownLimitThroughMultipleSort_AfterMvExpand2() { LogicalPlan plan = optimizedPlan(""" @@ -2119,10 +2111,7 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand2() { assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var mvExp = as(topN.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); - assertThat(orderNames(topN), contains("emp_no")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** @@ -2231,8 +2220,7 @@ public void testPushDown_TheRightLimit_PastLookupJoin() { * \_TopN[[Order[salary{f}#12,ASC,LAST]],5[INTEGER]] * \_Eval[[100[INTEGER] AS b]] * \_MvExpand[first_name{f}#11] - * \_TopN[[Order[first_name{f}#11,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[employees][emp_no{f}#10, first_name{f}#11, salary{f}#12] + * \_EsRelation[employees][emp_no{f}#10, first_name{f}#11, salary{f}#12] */ public void testPushDownLimit_PastEvalAndMvExpand() { LogicalPlan plan = optimizedPlan(""" @@ -2250,22 +2238,18 @@ public void testPushDownLimit_PastEvalAndMvExpand() { assertThat(orderNames(topN), contains("salary")); var eval = as(topN.child(), Eval.class); var mvExp = as(eval.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); - assertThat(orderNames(topN), contains("first_name")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** * Expected - * EsqlProject[[emp_no{f}#12, first_name{r}#22, salary{f}#17]] - * \_TopN[[Order[salary{f}#17,ASC,LAST], Order[first_name{r}#22,ASC,LAST]],1000[INTEGER]] - * \_Filter[gender{f}#14 == [46][KEYWORD] AND WILDCARDLIKE(first_name{r}#22)] - * \_MvExpand[first_name{f}#13,first_name{r}#22,null] - * \_TopN[[Order[emp_no{f}#12,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[test][_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, ..] - */ - public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField_ResultTruncationDefaultSize() { + * EsqlProject[[emp_no{f}#5885, first_name{r}#5896, salary{f}#5890]] + * \_TopN[[Order[salary{f}#5890,ASC,LAST], Order[first_name{r}#5896,ASC,LAST]],1000[INTEGER]] + * \_Filter[gender{f}#5887 == [46][KEYWORD] AND WILDCARDLIKE(first_name{r}#5896)] + * \_MvExpand[first_name{f}#5886,first_name{r}#5896,null] + * \_EsRelation[test][_meta_field{f}#5891, emp_no{f}#5885, first_name{f}#..] + */ + public void testRedundantSort_BeforeMvExpand_WithFilterOnExpandedField_ResultTruncationDefaultSize() { LogicalPlan plan = optimizedPlan(""" from test | sort emp_no @@ -2282,9 +2266,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField_ResultT var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); var mvExp = as(filter.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); // TODO is it correct? Double-check AddDefaultTopN rule - assertThat(orderNames(topN), contains("emp_no")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** @@ -2367,8 +2349,7 @@ public void testMultiMvExpand_SortDownBelow() { var mvExpand = as(topN.child(), MvExpand.class); var filter = as(mvExpand.child(), Filter.class); mvExpand = as(filter.child(), MvExpand.class); - var topN2 = as(mvExpand.child(), TopN.class); // TODO is it correct? Double-check AddDefaultTopN rule - as(topN2.child(), EsRelation.class); + as(mvExpand.child(), EsRelation.class); } /** @@ -2463,20 +2444,18 @@ public void testRemoveUnusedSortBeforeMvExpand_DefaultLimit10000() { assertThat(orderNames(topN), contains("first_name")); assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); var mvExpand = as(topN.child(), MvExpand.class); - var topN2 = as(mvExpand.child(), TopN.class); // TODO is it correct? Double-check AddDefaultTopN rule - as(topN2.child(), EsRelation.class); + as(mvExpand.child(), EsRelation.class); } /** * Expected - * EsqlProject[[emp_no{f}#104, first_name{f}#105, salary{f}#106]] - * \_TopN[[Order[salary{f}#106,ASC,LAST], Order[first_name{f}#105,ASC,LAST]],15[INTEGER]] - * \_Filter[gender{f}#215 == [46][KEYWORD] AND WILDCARDLIKE(first_name{f}#105)] - * \_MvExpand[first_name{f}#105] - * \_TopN[[Order[emp_no{f}#104,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[employees][emp_no{f}#104, first_name{f}#105, salary{f}#106] - */ - public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() { + * EsqlProject[[emp_no{f}#3517, first_name{r}#3528, salary{f}#3522]] + * \_TopN[[Order[salary{f}#3522,ASC,LAST], Order[first_name{r}#3528,ASC,LAST]],15[INTEGER]] + * \_Filter[gender{f}#3519 == [46][KEYWORD] AND WILDCARDLIKE(first_name{r}#3528)] + * \_MvExpand[first_name{f}#3518,first_name{r}#3528,null] + * \_EsRelation[test][_meta_field{f}#3523, emp_no{f}#3517, first_name{f}#..] + */ + public void testRedundantSort_BeforeMvExpand_WithFilterOnExpandedField() { LogicalPlan plan = optimizedPlan(""" from test | sort emp_no @@ -2494,24 +2473,18 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() { var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); var mvExp = as(filter.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); - // the filter acts on first_name (the one used in mv_expand), so the limit 15 is not pushed down past mv_expand - // instead the default limit is added - assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); - assertThat(orderNames(topN), contains("emp_no")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** * Expected - * EsqlProject[[emp_no{f}#104, first_name{f}#105, salary{f}#106]] - * \_TopN[[Order[salary{f}#106,ASC,LAST], Order[first_name{f}#105,ASC,LAST]],15[INTEGER]] - * \_Filter[gender{f}#215 == [46][KEYWORD] AND salary{f}#106 > 60000[INTEGER]] - * \_MvExpand[first_name{f}#105] - * \_TopN[[Order[emp_no{f}#104,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[employees][emp_no{f}#104, first_name{f}#105, salary{f}#106] - */ - public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField() { + * EsqlProject[[emp_no{f}#3421, first_name{r}#3432, salary{f}#3426]] + * \_TopN[[Order[salary{f}#3426,ASC,LAST], Order[first_name{r}#3432,ASC,LAST]],15[INTEGER]] + * \_Filter[gender{f}#3423 == [46][KEYWORD] AND salary{f}#3426 > 60000[INTEGER]] + * \_MvExpand[first_name{f}#3422,first_name{r}#3432,null] + * \_EsRelation[test][_meta_field{f}#3427, emp_no{f}#3421, first_name{f}#..] + */ + public void testRedundantSort_BeforeMvExpand_WithFilter_NOT_OnExpandedField() { LogicalPlan plan = optimizedPlan(""" from test | sort emp_no @@ -2529,24 +2502,18 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField() var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); var mvExp = as(filter.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); - // the filters after mv_expand do not act on the expanded field values, as such the limit 15 is the one being pushed down - // otherwise that limit wouldn't have pushed down and the default limit was instead being added by default before mv_expanded - assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); - assertThat(orderNames(topN), contains("emp_no")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** * Expected - * EsqlProject[[emp_no{f}#116, first_name{f}#117 AS x, salary{f}#119]] - * \_TopN[[Order[salary{f}#119,ASC,LAST], Order[first_name{f}#117,ASC,LAST]],15[INTEGER]] - * \_Filter[gender{f}#118 == [46][KEYWORD] AND WILDCARDLIKE(first_name{f}#117)] - * \_MvExpand[first_name{f}#117] - * \_TopN[[Order[gender{f}#118,ASC,LAST]],10000[INTEGER]] - * \_EsRelation[employees][emp_no{f}#116, first_name{f}#117, gender{f}#118, sa..] - */ - public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedFieldAlias() { + * EsqlProject[[emp_no{f}#2085, first_name{r}#2096 AS x, salary{f}#2090]] + * \_TopN[[Order[salary{f}#2090,ASC,LAST], Order[first_name{r}#2096,ASC,LAST]],15[INTEGER]] + * \_Filter[gender{f}#2087 == [46][KEYWORD] AND WILDCARDLIKE(first_name{r}#2096)] + * \_MvExpand[first_name{f}#2086,first_name{r}#2096,null] + * \_EsRelation[test][_meta_field{f}#2091, emp_no{f}#2085, first_name{f}#..] + */ + public void testRedundantSort_BeforeMvExpand_WithFilterOnExpandedFieldAlias() { LogicalPlan plan = optimizedPlan(""" from test | sort gender @@ -2565,11 +2532,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedFieldAlias() var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); var mvExp = as(filter.child(), MvExpand.class); - topN = as(mvExp.child(), TopN.class); - // the filter uses an alias ("x") to the expanded field ("first_name"), so the default limit is used and not the one provided - assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); - assertThat(orderNames(topN), contains("gender")); - as(topN.child(), EsRelation.class); + as(mvExp.child(), EsRelation.class); } /** @@ -7289,4 +7252,349 @@ public void testFunctionNamedParamsAsFunctionArgument() { assertEquals(new Literal(EMPTY, 2.0, DataType.DOUBLE), ee.value()); assertEquals(DataType.DOUBLE, ee.dataType()); } + + /** + * TopN[[Order[emp_no{f}#11,ASC,LAST]],1000[INTEGER]] + * \_Join[LEFT,[language_code{r}#5],[language_code{r}#5],[language_code{f}#22]] + * |_EsqlProject[[_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, gender{f}#13, hire_date{f}#18, job{f}#19, job.raw{f}#20, l + * anguages{f}#14 AS language_code, last_name{f}#15, long_noidx{f}#21, salary{f}#16, foo{r}#7]] + * | \_Eval[[[62 61 72][KEYWORD] AS foo]] + * | \_Filter[languages{f}#14 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#22, language_name{f}#23] + */ + public void testRedundantSortOnJoin() { + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + + var plan = optimizedPlan(""" + FROM test + | SORT languages + | RENAME languages AS language_code + | EVAL foo = "bar" + | LOOKUP JOIN languages_lookup ON language_code + | WHERE language_code > 1 + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var join = as(topN.child(), Join.class); + var project = as(join.left(), EsqlProject.class); + var eval = as(project.child(), Eval.class); + var filter = as(eval.child(), Filter.class); + as(filter.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#9,ASC,LAST]],1000[INTEGER]] + * \_Filter[emp_no{f}#9 > 1[INTEGER]] + * \_MvExpand[languages{f}#12,languages{r}#20,null] + * \_Eval[[[62 61 72][KEYWORD] AS foo]] + * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..] + */ + public void testRedundantSortOnMvExpand() { + var plan = optimizedPlan(""" + FROM test + | SORT languages + | EVAL foo = "bar" + | MV_EXPAND languages + | WHERE emp_no > 1 + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var filter = as(topN.child(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + as(eval.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#11,ASC,LAST]],1000[INTEGER]] + * \_Join[LEFT,[language_code{r}#5],[language_code{r}#5],[language_code{f}#22]] + * |_Filter[emp_no{f}#11 > 1[INTEGER]] + * | \_MvExpand[languages{f}#14,languages{r}#24,null] + * | \_Eval[[languages{f}#14 AS language_code]] + * | \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#22, language_name{f}#23] + */ + public void testRedundantSortOnMvExpandAndJoin() { + var plan = optimizedPlan(""" + FROM test + | SORT languages + | EVAL language_code = languages + | MV_EXPAND languages + | WHERE emp_no > 1 + | LOOKUP JOIN languages_lookup ON language_code + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var join = as(topN.child(), Join.class); + var filter = as(join.left(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + as(eval.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#12,ASC,LAST]],1000[INTEGER]] + * \_Join[LEFT,[language_code{r}#5],[language_code{r}#5],[language_code{f}#23]] + * |_Filter[emp_no{f}#12 > 1[INTEGER]] + * | \_MvExpand[languages{f}#15,languages{r}#25,null] + * | \_Eval[[languages{f}#15 AS language_code]] + * | \_EsRelation[test][_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, ..] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#23, language_name{f}#24] + */ + public void testMultlipleRedundantSortOnMvExpandAndJoin() { + var plan = optimizedPlan(""" + FROM test + | SORT first_name + | EVAL language_code = languages + | MV_EXPAND languages + | sort last_name + | WHERE emp_no > 1 + | LOOKUP JOIN languages_lookup ON language_code + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var join = as(topN.child(), Join.class); + var filter = as(join.left(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + as(eval.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#16,ASC,LAST]],1000[INTEGER]] + * \_Filter[emp_no{f}#16 > 1[INTEGER]] + * \_MvExpand[languages{f}#19,languages{r}#31] + * \_Dissect[foo{r}#5,Parser[pattern=%{z}, appendSeparator=, parser=org.elasticsearch.dissect.DissectParser@26f2cab],[z{r}#10 + * ]] + * \_Grok[foo{r}#5,Parser[pattern=%{WORD:y}, grok=org.elasticsearch.grok.Grok@6ea44ccd],[y{r}#9]] + * \_Enrich[ANY,[6c 61 6e 67 75 61 67 65 73 5f 69 64 78][KEYWORD],foo{r}#5,{"match":{"indices":[],"match_field":"id","enrich_ + * fields":["language_code","language_name"]}},{=languages_idx},[language_code{r}#29, language_name{r}#30]] + * \_Eval[[TOSTRING(languages{f}#19) AS foo]] + * \_EsRelation[test][_meta_field{f}#22, emp_no{f}#16, first_name{f}#17, ..] + */ + public void testRedundantSortOnMvExpandEnrichGrokDissect() { + var plan = optimizedPlan(""" + FROM test + | SORT languages + | EVAL foo = to_string(languages) + | ENRICH languages_idx on foo + | GROK foo "%{WORD:y}" + | DISSECT foo "%{z}" + | MV_EXPAND languages + | WHERE emp_no > 1 + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var filter = as(topN.child(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var dissect = as(mvExpand.child(), Dissect.class); + var grok = as(dissect.child(), Grok.class); + var enrich = as(grok.child(), Enrich.class); + var eval = as(enrich.child(), Eval.class); + as(eval.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#20,ASC,LAST]],1000[INTEGER]] + * \_Filter[emp_no{f}#20 > 1[INTEGER]] + * \_MvExpand[languages{f}#23,languages{r}#37] + * \_Dissect[foo{r}#5,Parser[pattern=%{z}, appendSeparator=, parser=org.elasticsearch.dissect.DissectParser@3e922db0],[z{r}#1 + * 4]] + * \_Grok[foo{r}#5,Parser[pattern=%{WORD:y}, grok=org.elasticsearch.grok.Grok@4d6ad024],[y{r}#13]] + * \_Enrich[ANY,[6c 61 6e 67 75 61 67 65 73 5f 69 64 78][KEYWORD],foo{r}#5,{"match":{"indices":[],"match_field":"id","enrich_ + * fields":["language_code","language_name"]}},{=languages_idx},[language_code{r}#35, language_name{r}#36]] + * \_Join[LEFT,[language_code{r}#8],[language_code{r}#8],[language_code{f}#31]] + * |_Eval[[TOSTRING(languages{f}#23) AS foo, languages{f}#23 AS language_code]] + * | \_EsRelation[test][_meta_field{f}#26, emp_no{f}#20, first_name{f}#21, ..] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#31] + */ + public void testRedundantSortOnMvExpandJoinEnrichGrokDissect() { + var plan = optimizedPlan(""" + FROM test + | SORT languages + | EVAL foo = to_string(languages), language_code = languages + | LOOKUP JOIN languages_lookup ON language_code + | ENRICH languages_idx on foo + | GROK foo "%{WORD:y}" + | DISSECT foo "%{z}" + | MV_EXPAND languages + | WHERE emp_no > 1 + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var filter = as(topN.child(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var dissect = as(mvExpand.child(), Dissect.class); + var grok = as(dissect.child(), Grok.class); + var enrich = as(grok.child(), Enrich.class); + var join = as(enrich.child(), Join.class); + var eval = as(join.left(), Eval.class); + as(eval.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#23,ASC,LAST]],1000[INTEGER]] + * \_Filter[emp_no{f}#23 > 1[INTEGER]] + * \_MvExpand[languages{f}#26,languages{r}#36] + * \_EsqlProject[[language_name{f}#35, foo{r}#5 AS bar, languages{f}#26, emp_no{f}#23]] + * \_Join[LEFT,[language_code{r}#8],[language_code{r}#8],[language_code{f}#34]] + * |_Project[[_meta_field{f}#29, emp_no{f}#23, first_name{f}#24, gender{f}#25, hire_date{f}#30, job{f}#31, job.raw{f}#32, l + * anguages{f}#26, last_name{f}#27, long_noidx{f}#33, salary{f}#28, foo{r}#5, languages{f}#26 AS language_code]] + * | \_Eval[[TOSTRING(languages{f}#26) AS foo]] + * | \_EsRelation[test][_meta_field{f}#29, emp_no{f}#23, first_name{f}#24, ..] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#34, language_name{f}#35] + */ + public void testRedundantSortOnMvExpandJoinKeepDropRename() { + var plan = optimizedPlan(""" + FROM test + | SORT languages + | EVAL foo = to_string(languages), language_code = languages + | LOOKUP JOIN languages_lookup ON language_code + | KEEP language_name, language_code, foo, languages, emp_no + | DROP language_code + | RENAME foo AS bar + | MV_EXPAND languages + | WHERE emp_no > 1 + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var filter = as(topN.child(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var project = as(mvExpand.child(), Project.class); + var join = as(project.child(), Join.class); + var project2 = as(join.left(), Project.class); + var eval = as(project2.child(), Eval.class); + as(eval.child(), EsRelation.class); + } + + /** + * TopN[[Order[emp_no{f}#15,ASC,LAST]],1000[INTEGER]] + * \_Filter[emp_no{f}#15 > 1[INTEGER]] + * \_MvExpand[foo{r}#10,foo{r}#29] + * \_Eval[[CONCAT(language_name{r}#28,[66 6f 6f][KEYWORD]) AS foo]] + * \_MvExpand[language_name{f}#27,language_name{r}#28] + * \_Join[LEFT,[language_code{r}#3],[language_code{r}#3],[language_code{f}#26]] + * |_Eval[[1[INTEGER] AS language_code]] + * | \_EsRelation[test][_meta_field{f}#21, emp_no{f}#15, first_name{f}#16, ..] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#26, language_name{f}#27] + */ + public void testEvalLookupMultipleSorts() { + var plan = optimizedPlan(""" + FROM test + | EVAL language_code = 1 + | LOOKUP JOIN languages_lookup ON language_code + | SORT language_name + | MV_EXPAND language_name + | EVAL foo = concat(language_name, "foo") + | MV_EXPAND foo + | WHERE emp_no > 1 + | SORT emp_no + """); + + var topN = as(plan, TopN.class); + var filter = as(topN.child(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + mvExpand = as(eval.child(), MvExpand.class); + var join = as(mvExpand.child(), Join.class); + eval = as(join.left(), Eval.class); + as(eval.child(), EsRelation.class); + + } + + public void testUnboundedSortSimple() { + var query = """ + ROW x = [1,2,3], y = 1 + | SORT y + | MV_EXPAND x + | WHERE x > 2 + """; + + VerificationException e = expectThrows(VerificationException.class, () -> plan(query)); + assertThat(e.getMessage(), containsString("line 2:5: Unbounded sort not supported yet [SORT y] please add a limit")); + } + + public void testUnboundedSortJoin() { + var query = """ + ROW x = [1,2,3], y = 2, language_code = 1 + | SORT y + | LOOKUP JOIN languages_lookup ON language_code + | WHERE language_name == "foo" + """; + + VerificationException e = expectThrows(VerificationException.class, () -> plan(query)); + assertThat(e.getMessage(), containsString("line 2:5: Unbounded sort not supported yet [SORT y] please add a limit")); + } + + public void testUnboundedSortWithMvExpandAndFilter() { + var query = """ + FROM test + | EVAL language_code = 1 + | LOOKUP JOIN languages_lookup ON language_code + | SORT language_name + | EVAL foo = concat(language_name, "foo") + | MV_EXPAND foo + | WHERE foo == "foo" + """; + + VerificationException e = expectThrows(VerificationException.class, () -> plan(query)); + assertThat(e.getMessage(), containsString("line 4:3: Unbounded sort not supported yet [SORT language_name] please add a limit")); + } + + public void testUnboundedSortWithLookupJoinAndFilter() { + var query = """ + FROM test + | EVAL language_code = 1 + | EVAL foo = concat(language_code::string, "foo") + | MV_EXPAND foo + | SORT foo + | LOOKUP JOIN languages_lookup ON language_code + | WHERE language_name == "foo" + """; + + VerificationException e = expectThrows(VerificationException.class, () -> plan(query)); + assertThat(e.getMessage(), containsString("line 5:3: Unbounded sort not supported yet [SORT foo] please add a limit")); + } + + public void testUnboundedSortExpandFilter() { + var query = """ + ROW x = [1,2,3], y = 1 + | SORT x + | MV_EXPAND x + | WHERE x > 2 + """; + + VerificationException e = expectThrows(VerificationException.class, () -> plan(query)); + assertThat(e.getMessage(), containsString("line 2:5: Unbounded sort not supported yet [SORT x] please add a limit")); + } + + public void testPruneRedundantOrderBy() { + var rule = new PruneRedundantOrderBy(); + + var query = """ + row x = [1,2,3], y = 1 + | sort x + | mv_expand x + | sort x + | mv_expand x + | sort y + """; + LogicalPlan analyzed = analyzer.analyze(parser.createStatement(query)); + LogicalPlan optimized = rule.apply(analyzed); + + // check that all the redundant SORTs are removed in a single run + var limit = as(optimized, Limit.class); + var orderBy = as(limit.child(), OrderBy.class); + var mvExpand = as(orderBy.child(), MvExpand.class); + var mvExpand2 = as(mvExpand.child(), MvExpand.class); + as(mvExpand2.child(), Row.class); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 3d5159a842411..3e417440c4fed 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; @@ -218,7 +219,10 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { private TestDataSource airportsNotIndexed; // Test when spatial field has doc values but is not indexed private TestDataSource airportsNotIndexedNorDocValues; // Test when spatial field is neither indexed nor has doc-values private TestDataSource airportsWeb; // Cartesian point field tests - private TestDataSource airportsCityBoundaries; + private TestDataSource airportsCityBoundaries; // geo_shape field tests + private TestDataSource airportsCityBoundariesNoPointDocValues; // Disable doc-values on geo_point fields, but not geo_shape fields + private TestDataSource airportsCityBoundariesNoShapeDocValues; // Disable doc-values on geo_shape fields, but not geo_point fields + private TestDataSource airportsCityBoundariesNoDocValues; // Dsiable doc-values on both geo_point and geo_shape fields private TestDataSource cartesianMultipolygons; // cartesian_shape field tests private TestDataSource cartesianMultipolygonsNoDocValues; // cartesian_shape field tests but has no doc values private TestDataSource countriesBbox; // geo_shape field tests @@ -296,6 +300,27 @@ public void init() { functionRegistry, enrichResolution ); + this.airportsCityBoundariesNoPointDocValues = makeTestDataSource( + "airports_city_boundaries", + "mapping-airport_city_boundaries.json", + functionRegistry, + enrichResolution, + new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "location", "city_location") + ); + this.airportsCityBoundariesNoShapeDocValues = makeTestDataSource( + "airports_city_boundaries", + "mapping-airport_city_boundaries.json", + functionRegistry, + enrichResolution, + new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "city_boundary") + ); + this.airportsCityBoundariesNoDocValues = makeTestDataSource( + "airports_city_boundaries", + "mapping-airport_city_boundaries.json", + functionRegistry, + enrichResolution, + new TestConfigurableSearchStats().exclude(Config.DOC_VALUES, "city_boundary", "location", "city_location") + ); this.cartesianMultipolygons = makeTestDataSource( "cartesian_multipolygons", "mapping-cartesian_multipolygons.json", @@ -1107,6 +1132,71 @@ public void testPushMultipleBinaryLogicFilters() { assertThat(rq.to(), nullValue()); } + /** + * Expects + * + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, gender{f}#7, + * hire_date{f}#12, job{f}#13, job.raw{f}#14, languages{f}#8, last_name{f}#9, + * long_noidx{f}#15, salary{f}#10],false] + * \_ProjectExec[[_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, gender{f}#7, + * hire_date{f}#12, job{f}#13, job.raw{f}#14, languages{f}#8, last_name{f}#9, + * long_noidx{f}#15, salary{f}#10]] + * \_FieldExtractExec[_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..] + * \_EsQueryExec[test], indexMode[standard], query[ + * {"bool":{"must":[ + * {"bool":{"should":[ + * {"esql_single_value":{"field":"first_name","next": + * {"wildcard":{"first_name":{"wildcard":"\\*Firs*","boost":1.0}}} + * "source":"starts_with(first_name, \"*Firs\")@2:9"}}, + * {"esql_single_value":{"field":"first_name","next": + * {"wildcard":{"first_name":{"wildcard":"*irst\\*","boost":1.0}}}, + * "source":"ends_with(first_name, \"irst*\")@2:45"}}],"boost":1.0}}, + * {"esql_single_value":{"field":"last_name","next": + * {"wildcard":{"last_name":{"wildcard":"*ast","boost":1.0}}}, + * "source":"ends_with(last_name, \"ast\")@3:9"}} + * ],"boost":1.0}} + * ][_doc{f}#27], limit[1000], sort[] estimatedRowSize[332] + */ + public void testPushMultipleFunctions() { + var plan = physicalPlan(""" + from airports + | where starts_with(first_name, "*Firs") or ends_with(first_name, "irst*") + | where ends_with(last_name, "ast") + """); + + var optimized = optimizedPlan(plan); + var topLimit = as(optimized, LimitExec.class); + var exchange = asRemoteExchange(topLimit.child()); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var source = source(fieldExtract.child()); + assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES)); + + var andBool = as(source.query(), BoolQueryBuilder.class); + assertThat(andBool.must(), hasSize(2)); + assertThat(andBool.should(), hasSize(0)); + + var orBool = as(andBool.must().get(0), BoolQueryBuilder.class); + assertThat(orBool.should(), hasSize(2)); + assertThat(orBool.must(), hasSize(0)); + + var orStartsWith = as(sv(orBool.should().get(0), "first_name"), WildcardQueryBuilder.class); + assertThat(orStartsWith.fieldName(), equalTo("first_name")); + assertThat(orStartsWith.caseInsensitive(), equalTo(false)); + assertThat(orStartsWith.value(), equalTo("\\*Firs*")); + + var orEndsWith = as(sv(orBool.should().get(1), "first_name"), WildcardQueryBuilder.class); + assertThat(orEndsWith.fieldName(), equalTo("first_name")); + assertThat(orEndsWith.caseInsensitive(), equalTo(false)); + assertThat(orEndsWith.value(), equalTo("*irst\\*")); + + var andEndsWith = as(sv(andBool.must().get(1), "last_name"), WildcardQueryBuilder.class); + assertThat(andEndsWith.fieldName(), equalTo("last_name")); + assertThat(andEndsWith.caseInsensitive(), equalTo(false)); + assertThat(andEndsWith.value(), equalTo("*ast")); + } + public void testLimit() { var optimized = optimizedPlan(physicalPlan(""" from test @@ -3279,39 +3369,39 @@ public void testSpatialTypesAndStatsExtentAndCentroidUseDocValues() { * ][_doc{f}#36], limit[], sort[] estimatedRowSize[204] * */ - public void testSpatialTypesAndStatsExtentOfGeoShapeDoesNotUseBinaryExtraction() { - // TODO: When we get geo_shape working with bounds extraction from doc-values, change the name of this test + public void testSpatialTypesAndStatsExtentOfGeoShapeUsesBinaryExtraction() { var query = "FROM airports_city_boundaries | STATS extent = ST_EXTENT_AGG(city_boundary)"; - var testData = airportsCityBoundaries; - var plan = physicalPlan(query, testData); + for (boolean useDocValues : new Boolean[] { true, false }) { + var testData = useDocValues ? airportsCityBoundaries : airportsCityBoundariesNoDocValues; + var plan = physicalPlan(query, testData); - var limit = as(plan, LimitExec.class); - var agg = as(limit.child(), AggregateExec.class); - // Before optimization the aggregation does not use extent extraction - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + // Before optimization the aggregation does not use extent extraction + assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - var exchange = as(agg.child(), ExchangeExec.class); - var fragment = as(exchange.child(), FragmentExec.class); - var fAgg = as(fragment.fragment(), Aggregate.class); - as(fAgg.child(), EsRelation.class); + var exchange = as(agg.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var fAgg = as(fragment.fragment(), Aggregate.class); + as(fAgg.child(), EsRelation.class); - // Now optimize the plan and assert the aggregation uses extent extraction - var optimized = optimizedPlan(plan, testData.stats); - limit = as(optimized, LimitExec.class); - agg = as(limit.child(), AggregateExec.class); - // Above the exchange (in coordinator) the aggregation is not using doc-values - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - exchange = as(agg.child(), ExchangeExec.class); - agg = as(exchange.child(), AggregateExec.class); - // below the exchange (in data node) the aggregation is using a specific - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - assertChildIsExtractedAs(agg, FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS, GEO_SHAPE); + // Now optimize the plan and assert the aggregation uses extent extraction + var optimized = optimizedPlan(plan, testData.stats); + limit = as(optimized, LimitExec.class); + agg = as(limit.child(), AggregateExec.class); + // Above the exchange (in coordinator) the aggregation is not using doc-values + assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + exchange = as(agg.child(), ExchangeExec.class); + agg = as(exchange.child(), AggregateExec.class); + // below the exchange (in data node) the aggregation is using a specific int[] which the aggregation needs to know about. + var fieldExtractPreference = useDocValues ? FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS : FieldExtractPreference.NONE; + assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, fieldExtractPreference); + assertChildIsExtractedAs(agg, fieldExtractPreference, GEO_SHAPE); + } } /** * This test verifies that the aggregation does not use spatial bounds extraction when the shape appears in an eval or filter. - * TODO: Currently this tests nothing, because geo_shape is not supported anyway for bounds extraction, - * but it should be updated when it is supported. */ public void testSpatialTypesAndStatsExtentOfShapesNegativeCases() { for (String query : new String[] { """ @@ -3334,6 +3424,7 @@ public void testSpatialTypesAndStatsExtentOfShapesNegativeCases() { assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); var exchange = as(agg.child(), ExchangeExec.class); agg = as(exchange.child(), AggregateExec.class); + // Because the shape was used in EVAL/WHERE we cannot use doc-values bounds extraction optimization assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); var exec = agg.child() instanceof FieldExtractExec ? agg : as(agg.child(), UnaryExec.class); assertChildIsExtractedAs(exec, FieldExtractPreference.NONE, GEO_SHAPE); @@ -3359,19 +3450,11 @@ public void testSpatialTypesAndStatsExtentOfCartesianShapesWithAndWithoutDocValu var optimized = optimizedPlan(plan, testData.stats); limit = as(optimized, LimitExec.class); agg = as(limit.child(), AggregateExec.class); - // For cartesian_shape extraction, we extract bounds from doc-values directly into a BBOX encoded as BytesRef, - // so the aggregation does not need to know about it. assertAggregation(agg, "extent", SpatialExtent.class, CARTESIAN_SHAPE, FieldExtractPreference.NONE); var exchange = as(agg.child(), ExchangeExec.class); agg = as(exchange.child(), AggregateExec.class); - assertAggregation( - agg, - "extent", - "hasDocValues:" + hasDocValues, - SpatialExtent.class, - CARTESIAN_SHAPE, - FieldExtractPreference.NONE - ); + // We extract bounds from doc-values into a special int[] which the aggregation needs to know about. + assertAggregation(agg, "extent", "hasDocValues:" + hasDocValues, SpatialExtent.class, CARTESIAN_SHAPE, fieldExtractPreference); var exec = agg.child() instanceof FieldExtractExec ? agg : as(agg.child(), UnaryExec.class); // For cartesian_shape, the bounds extraction is done in the FieldExtractExec, so it does need to know about this assertChildIsExtractedAs(exec, fieldExtractPreference, CARTESIAN_SHAPE); @@ -3379,60 +3462,72 @@ public void testSpatialTypesAndStatsExtentOfCartesianShapesWithAndWithoutDocValu } /** - * Before local optimizations: + * This tests all four combinations of geo_point and geo_shape with and without doc-values. + * Since each will be extracted differently (points as encoded longs, and shapes as int[5] bounds representing Extents), + * we want to verify that the combinations do not clash and work together. + * The optimized query plan in the case when both points and shapes have doc-values will look like: * * LimitExec[1000[INTEGER]] - * \_AggregateExec[[],[SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, SPATIALCENTROID(city_location{f}#12,true[BOOLEA - * N]) AS centroid],...] - * \_ExchangeExec[[..]] - * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[...]] - * \_EsRelation[airports_city_boundaries][abbrev{f}#8, airport{f}#9, city{f}#11, city_boundar..] - * - * After local optimizations: - * - * LimitExec[1000[INTEGER]] - * \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],FINAL,[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, - * maxPosX{r}#55, maxY{r}#56, minY{r}#57],21] - * \_ExchangeExec[[minNegX{r}#52, minPosX{r}#53, maxNegX{r}#54, maxPosX{r}#55, maxY{r}#56, minY{r}#57],true] - * \_AggregateExec[[],[SPATIALSTEXTENT(location{f}#48,true[BOOLEAN]) AS extent],INITIAL,[ - * minNegX{r}#73, minPosX{r}#74, maxNegX{rb#75, maxPosX{r}#76, maxY{r}#77, minY{r}#78],21] - * \_FieldExtractExec[location{f}#48][location{f}#48] - * \_EsQueryExec[airports], indexMode[standard], query[{"exists":{"field":"location","boost":1.0}}][ - * _doc{f}#79], limit[], sort[] estimatedRowSize[25] + * \_AggregateExec[[],[ + * SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, + * SPATIALCENTROID(city_location{f}#12,true[BOOLEAN]) AS centroid + * ],FINAL,[...bounds attributes..., ...centroid attributes...],221] + * \_ExchangeExec[[...bounds attributes..., ...centroid attributes...],true] + * \_AggregateExec[[],[ + * SPATIALEXTENT(city_boundary{f}#13,true[BOOLEAN]) AS extent, + * SPATIALCENTROID(city_location{f}#12,true[BOOLEAN]) AS centroid + * ],INITIAL,[...bounds attributes..., ...centroid attributes...],221] + * \_FieldExtractExec[city_boundary{f}#13, city_location{f}#12][city_location{f}#12],[city_boundary{f}#13] + * \_EsQueryExec[airports_city_boundaries], indexMode[standard], query[ + * {"bool":{"should":[ + * {"exists":{"field":"city_boundary","boost":1.0}}, + * {"exists":{"field":"city_location","boost":1.0}} + * ],"boost":1.0}} + * ][_doc{f}#55], limit[], sort[] estimatedRowSize[225] * */ public void testMixedSpatialBoundsAndPointsExtracted() { var query = """ FROM airports_city_boundaries \ | STATS extent = ST_EXTENT_AGG(city_boundary), centroid = ST_CENTROID_AGG(city_location)"""; - var testData = airportsCityBoundaries; - var plan = physicalPlan(query, testData); + for (boolean pointDocValues : new Boolean[] { true, false }) { + for (boolean shapeDocValues : new Boolean[] { true, false }) { + var testData = pointDocValues + ? (shapeDocValues ? airportsCityBoundaries : airportsCityBoundariesNoShapeDocValues) + : (shapeDocValues ? airportsCityBoundariesNoPointDocValues : airportsCityBoundariesNoDocValues); + var msg = "DocValues[point:" + pointDocValues + ", shape:" + shapeDocValues + "]"; + var plan = physicalPlan(query, testData); - var limit = as(plan, LimitExec.class); - var agg = as(limit.child(), AggregateExec.class); - // Before optimization the aggregation does not use doc-values - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + // Before optimization the aggregation does not use doc-values + assertAggregation(agg, "extent", msg, SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + assertAggregation(agg, "centroid", msg, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); - var exchange = as(agg.child(), ExchangeExec.class); - var fragment = as(exchange.child(), FragmentExec.class); - var fAgg = as(fragment.fragment(), Aggregate.class); - as(fAgg.child(), EsRelation.class); + var exchange = as(agg.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var fAgg = as(fragment.fragment(), Aggregate.class); + as(fAgg.child(), EsRelation.class); - // Now optimize the plan and assert the aggregation uses both doc-values and bounds extraction - var optimized = optimizedPlan(plan, testData.stats); - limit = as(optimized, LimitExec.class); - agg = as(limit.child(), AggregateExec.class); - // Above the exchange (in coordinator) the aggregation is not field-optimized. - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - assertAggregation(agg, "centroid", SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); - exchange = as(agg.child(), ExchangeExec.class); - agg = as(exchange.child(), AggregateExec.class); - // below the exchange (in data node) the aggregation is field optimized. - assertAggregation(agg, "extent", SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); - var fieldExtractExec = as(agg.child(), FieldExtractExec.class); - assertThat(fieldExtractExec.boundsAttributes().stream().map(a -> a.sourceText()).toList(), equalTo(List.of("city_boundary"))); - assertThat(fieldExtractExec.docValuesAttributes().stream().map(a -> a.sourceText()).toList(), equalTo(List.of("city_location"))); + // Now optimize the plan and assert the aggregation uses both doc-values and bounds extraction + var optimized = optimizedPlan(plan, testData.stats); + limit = as(optimized, LimitExec.class); + agg = as(limit.child(), AggregateExec.class); + // Above the exchange (in coordinator) the aggregation is not field-optimized. + assertAggregation(agg, "extent", msg, SpatialExtent.class, GEO_SHAPE, FieldExtractPreference.NONE); + assertAggregation(agg, "centroid", msg, SpatialCentroid.class, GEO_POINT, FieldExtractPreference.NONE); + exchange = as(agg.child(), ExchangeExec.class); + agg = as(exchange.child(), AggregateExec.class); + var fieldExtractExec = as(agg.child(), FieldExtractExec.class); + // below the exchange (in data node) the aggregation is field optimized. + var shapeExtractPreference = shapeDocValues ? FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS : FieldExtractPreference.NONE; + assertAggregation(agg, "extent", msg, SpatialExtent.class, GEO_SHAPE, shapeExtractPreference); + List boundsAttributes = shapeDocValues ? List.of("city_boundary") : List.of(); + List docValuesAttributes = pointDocValues ? List.of("city_location") : List.of(); + assertThat(fieldExtractExec.boundsAttributes().stream().map(Node::sourceText).toList(), equalTo(boundsAttributes)); + assertThat(fieldExtractExec.docValuesAttributes().stream().map(Node::sourceText).toList(), equalTo(docValuesAttributes)); + } + } } /** @@ -7469,7 +7564,7 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP TestBlockFactory.getNonBreakingInstance(), Settings.EMPTY, config, - new ExchangeSourceHandler(10, null, null)::createExchangeSource, + new ExchangeSourceHandler(10, null)::createExchangeSource, () -> exchangeSinkHandler.createExchangeSink(() -> {}), null, null, @@ -7656,7 +7751,7 @@ private static void assertAggregation( var aggFunc = assertAggregation(plan, aliasName, aggClass); var aggField = as(aggFunc.field(), Attribute.class); var spatialAgg = as(aggFunc, SpatialAggregateFunction.class); - assertThat(spatialAgg.fieldExtractPreference(), equalTo(fieldExtractPreference)); + assertThat(reason, spatialAgg.fieldExtractPreference(), equalTo(fieldExtractPreference)); assertThat(reason, aggField.dataType(), equalTo(fieldType)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java index 18d8bc9fb0a75..2ca1d8c4d1288 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java @@ -19,7 +19,7 @@ public void testDevelopmentInline() throws Exception { } public void testDevelopmentLookup() throws Exception { - parse("row a = 1 | lookup \"foo\" on j", "lookup"); + parse("row a = 1 | lookup_\uD83D\uDC14 \"foo\" on j", "lookup_\uD83D\uDC14"); } public void testDevelopmentMetrics() throws Exception { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 01d171418f2c7..49f645defc626 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -1080,15 +1080,28 @@ public void testEnrich() { processingCommand("enrich _" + mode.name() + ":countries ON country_code") ); - expectError("from a | enrich countries on foo* ", "Using wildcards [*] in ENRICH WITH projections is not allowed [foo*]"); - expectError("from a | enrich countries on foo with bar*", "Using wildcards [*] in ENRICH WITH projections is not allowed [bar*]"); + expectError("from a | enrich countries on foo* ", "Using wildcards [*] in ENRICH WITH projections is not allowed, found [foo*]"); + expectError("from a | enrich countries on * ", "Using wildcards [*] in ENRICH WITH projections is not allowed, found [*]"); + expectError( + "from a | enrich countries on foo with bar*", + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [bar*]" + ); + expectError("from a | enrich countries on foo with *", "Using wildcards [*] in ENRICH WITH projections is not allowed, found [*]"); expectError( "from a | enrich countries on foo with x = bar* ", - "Using wildcards [*] in ENRICH WITH projections is not allowed [bar*]" + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [bar*]" + ); + expectError( + "from a | enrich countries on foo with x = * ", + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [*]" ); expectError( "from a | enrich countries on foo with x* = bar ", - "Using wildcards [*] in ENRICH WITH projections is not allowed [x*]" + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [x*]" + ); + expectError( + "from a | enrich countries on foo with * = bar ", + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [*]" ); expectError( "from a | enrich typo:countries on foo", @@ -1587,10 +1600,6 @@ public void testIntervalParam() { } public void testParamForIdentifier() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand // eval, where assertEquals( @@ -1848,10 +1857,6 @@ public void testParamForIdentifier() { } public void testParamForIdentifierPattern() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // name patterns can appear in keep and drop // all patterns LogicalPlan plan = statement( @@ -1941,10 +1946,6 @@ public void testParamForIdentifierPattern() { } public void testParamInInvalidPosition() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // param for pattern is not supported in eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand // where/stats/sort/dissect/grok are covered in RestEsqlTestCase List invalidParamPositions = List.of("eval ?f1 = 1", "stats x = ?f1(*)", "mv_expand ?f1", "rename ?f1 as ?f2"); @@ -1985,7 +1986,7 @@ public void testParamInInvalidPosition() { expectError( "from idx1 | " + enrich, List.of(paramAsPattern("f1", pattern), paramAsIdentifier("f2", "f.2"), paramAsIdentifier("f3", "f.3*")), - "Using wildcards [*] in ENRICH WITH projections is not allowed [" + pattern + "]" + "Using wildcards [*] in ENRICH WITH projections is not allowed, found [" + pattern + "]" ); expectError( "from idx1 | " + enrich, @@ -1996,10 +1997,6 @@ public void testParamInInvalidPosition() { } public void testMissingParam() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // cover all processing commands eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand/keep/drop String error = "Unknown query parameter [f1], did you mean [f4]?"; String errorMvExpandFunctionNameCommandOption = "Query parameter [?f1] is null or undefined, cannot be used as an identifier"; @@ -2465,8 +2462,25 @@ by fn2(f3, {"option1":["string1","string2"],"option2":[1,2,3],"option3":2.0,"opt assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); UnresolvedRelation ur = as(dissect.child(), UnresolvedRelation.class); assertEquals(ur, relation("test")); + } + public void testNamedFunctionArgumentInMapWithNamedParameters() { // map entry values provided in named parameter, arrays are not supported by named parameters yet + LinkedHashMap expectedMap1 = new LinkedHashMap<>(4); + expectedMap1.put("option1", "string"); + expectedMap1.put("option2", 1); + expectedMap1.put("option3", List.of(2.0, 3.0, 4.0)); + expectedMap1.put("option4", List.of(true, false)); + LinkedHashMap expectedMap2 = new LinkedHashMap<>(4); + expectedMap2.put("option1", List.of("string1", "string2")); + expectedMap2.put("option2", List.of(1, 2, 3)); + expectedMap2.put("option3", 2.0); + expectedMap2.put("option4", true); + LinkedHashMap expectedMap3 = new LinkedHashMap<>(4); + expectedMap3.put("option1", "string"); + expectedMap3.put("option2", 2.0); + expectedMap3.put("option3", List.of(1, 2, 3)); + expectedMap3.put("option4", List.of(true, false)); assertEquals( new Filter( EMPTY, @@ -2564,7 +2578,7 @@ by fn2(f3, {"option1":["string1","string2"],"option2":[1,2,3],"option3":2.0,"opt ) ); - plan = statement( + LogicalPlan plan = statement( """ from test | dissect ?fn1(?n1, ?n2, {"option1":?n3,"option2":?n4,"option3":[2.0,3.0,4.0],"option4":[true,false]}) "%{bar}" @@ -2584,16 +2598,16 @@ by fn2(f3, {"option1":["string1","string2"],"option2":[1,2,3],"option3":2.0,"opt ) ) ); - grok = as(plan, Grok.class); + Grok grok = as(plan, Grok.class); assertEquals(function("fn2", List.of(attribute("f3"), mapExpression(expectedMap2))), grok.input()); assertEquals("%{WORD:foo}", grok.parser().pattern()); assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); - dissect = as(grok.child(), Dissect.class); + Dissect dissect = as(grok.child(), Dissect.class); assertEquals(function("fn1", List.of(attribute("f1"), attribute("f2"), mapExpression(expectedMap1))), dissect.input()); assertEquals("%{bar}", dissect.parser().pattern()); assertEquals("", dissect.parser().appendSeparator()); assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); - ur = as(dissect.child(), UnresolvedRelation.class); + UnresolvedRelation ur = as(dissect.child(), UnresolvedRelation.class); assertEquals(ur, relation("test")); } @@ -2952,4 +2966,11 @@ public void testInvalidJoinPatterns() { ); } } + + public void testUnclosedParenthesis() { + String[] queries = { "row ]", "from source | eval x = [1,2,3]]" }; + for (String q : queries) { + expectError(q, "Invalid query"); + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/OrderExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/OrderExecSerializationTests.java deleted file mode 100644 index 755f1cd4f52da..0000000000000 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/OrderExecSerializationTests.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.plan.physical; - -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.Order; -import org.elasticsearch.xpack.esql.expression.OrderSerializationTests; - -import java.io.IOException; -import java.util.List; - -public class OrderExecSerializationTests extends AbstractPhysicalPlanSerializationTests { - public static OrderExec randomOrderExec(int depth) { - Source source = randomSource(); - PhysicalPlan child = randomChild(depth); - List order = randomList(1, 10, OrderSerializationTests::randomOrder); - return new OrderExec(source, child, order); - } - - @Override - protected OrderExec createTestInstance() { - return randomOrderExec(0); - } - - @Override - protected OrderExec mutateInstance(OrderExec instance) throws IOException { - PhysicalPlan child = instance.child(); - List order = instance.order(); - if (randomBoolean()) { - child = randomValueOtherThan(child, () -> randomChild(0)); - } else { - order = randomValueOtherThan(order, () -> randomList(1, 10, OrderSerializationTests::randomOrder)); - } - return new OrderExec(instance.source(), child, order); - } - - @Override - protected boolean alwaysEmptySource() { - return true; - } -} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 57ef2dd151958..780045077f7b8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -21,6 +21,7 @@ import org.elasticsearch.compute.data.DocBlock; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -34,9 +35,14 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.GeometryValidator; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.geometry.utils.WellKnownBinary; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -68,6 +74,9 @@ import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween; import static java.util.stream.Collectors.joining; import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir; +import static org.elasticsearch.compute.aggregation.spatial.SpatialAggregationUtils.encodeLongitude; +import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; +import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders { private final List indexPages; @@ -103,13 +112,7 @@ public PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fieldExt PhysicalOperation op = source; for (Attribute attr : fieldExtractExec.attributesToExtract()) { layout.append(attr); - op = op.with( - new TestFieldExtractOperatorFactory( - attr, - PlannerUtils.extractPreference(fieldExtractExec.docValuesAttributes().contains(attr)) - ), - layout.build() - ); + op = op.with(new TestFieldExtractOperatorFactory(attr, fieldExtractExec.fieldExtractPreference(attr)), layout.build()); } return op; } @@ -397,17 +400,16 @@ private Block extractBlockForColumn( FieldExtractPreference extractPreference, BiFunction extractBlock ) { - BlockFactory blockFactory = docBlock.blockFactory(); - boolean mapToDocValues = shouldMapToDocValues(dataType, extractPreference); try ( - Block.Builder blockBuilder = mapToDocValues - ? blockFactory.newLongBlockBuilder(docBlock.getPositionCount()) - : blockBuilder(dataType, docBlock.getPositionCount(), TestBlockFactory.getNonBreakingInstance()) + Block.Builder blockBuilder = blockBuilder( + dataType, + extractPreference, + docBlock.getPositionCount(), + TestBlockFactory.getNonBreakingInstance() + ) ) { foreachIndexDoc(docBlock, indexDoc -> { - TestBlockCopier blockCopier = mapToDocValues - ? TestSpatialPointStatsBlockCopier.create(indexDoc.asVector().docs(), dataType) - : new TestBlockCopier(indexDoc.asVector().docs()); + TestBlockCopier blockCopier = blockCopier(dataType, extractPreference, indexDoc.asVector().docs()); Block blockForIndex = extractBlock.apply(indexDoc, blockCopier); blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount()); }); @@ -418,10 +420,6 @@ private Block extractBlockForColumn( } } - private boolean shouldMapToDocValues(DataType dataType, FieldExtractPreference extractPreference) { - return extractPreference == FieldExtractPreference.DOC_VALUES && DataType.isSpatialPoint(dataType); - } - private static class TestBlockCopier { protected final IntVector docIndices; @@ -447,7 +445,6 @@ protected Block copyBlock(Block originalData) { /** * geo_point and cartesian_point are normally loaded as WKT from source, but for aggregations we can load them as doc-values * which are encoded Long values. This class is used to convert the test loaded WKB into encoded longs for the aggregators. - * TODO: We need a different solution to support geo_shape and cartesian_shape */ private abstract static class TestSpatialPointStatsBlockCopier extends TestBlockCopier { @@ -465,15 +462,15 @@ protected Block copyBlock(Block originalData) { for (int c = 0; c < docIndices.getPositionCount(); c++) { int doc = docIndices.getInt(c); int count = bytesRefBlock.getValueCount(doc); - int i = bytesRefBlock.getFirstValueIndex(doc); if (count == 0) { builder.appendNull(); } else { if (count > 1) { builder.beginPositionEntry(); } - for (int v = 0; v < count; v++) { - builder.appendLong(encode(bytesRefBlock.getBytesRef(i + v, scratch))); + int firstValueIndex = bytesRefBlock.getFirstValueIndex(doc); + for (int i = firstValueIndex; i < firstValueIndex + count; i++) { + builder.appendLong(encode(bytesRefBlock.getBytesRef(i, scratch))); } if (count > 1) { builder.endPositionEntry(); @@ -499,12 +496,123 @@ protected long encode(BytesRef wkb) { } } - private static Block.Builder blockBuilder(DataType dataType, int estimatedSize, BlockFactory blockFactory) { + /** + * geo_shape and cartesian_shape are normally loaded as WKT from source, but for ST_EXTENT_AGG we can load them from doc-values + * extracting the spatial Extent information. This class is used to convert the test loaded WKB into the int[6] used in the aggregators. + */ + private abstract static class TestSpatialShapeExtentBlockCopier extends TestBlockCopier { + protected final SpatialEnvelopeVisitor.PointVisitor pointVisitor; + private final SpatialEnvelopeVisitor visitor; + + private TestSpatialShapeExtentBlockCopier(IntVector docIndices, SpatialEnvelopeVisitor.PointVisitor pointVisitor) { + super(docIndices); + this.pointVisitor = pointVisitor; + this.visitor = new SpatialEnvelopeVisitor(pointVisitor); + } + + @Override + protected Block copyBlock(Block originalData) { + BytesRef scratch = new BytesRef(100); + BytesRefBlock bytesRefBlock = (BytesRefBlock) originalData; + try (IntBlock.Builder builder = bytesRefBlock.blockFactory().newIntBlockBuilder(docIndices.getPositionCount())) { + for (int c = 0; c < docIndices.getPositionCount(); c++) { + int doc = docIndices.getInt(c); + int count = bytesRefBlock.getValueCount(doc); + if (count == 0) { + builder.appendNull(); + } else { + pointVisitor.reset(); + int firstValueIndex = bytesRefBlock.getFirstValueIndex(doc); + for (int i = firstValueIndex; i < firstValueIndex + count; i++) { + BytesRef wkb = bytesRefBlock.getBytesRef(i, scratch); + Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length); + geometry.visit(visitor); + } + encodeExtent(builder); + } + } + return builder.build(); + } + } + + protected abstract void encodeExtent(IntBlock.Builder builder); + + private static TestSpatialShapeExtentBlockCopier create(IntVector docIndices, DataType dataType) { + return switch (dataType) { + case GEO_SHAPE -> new TestGeoCopier(docIndices); + case CARTESIAN_SHAPE -> new TestCartesianCopier(docIndices); + default -> throw new IllegalArgumentException("Unsupported spatial data type: " + dataType); + }; + } + + private static class TestGeoCopier extends TestSpatialShapeExtentBlockCopier { + private TestGeoCopier(IntVector docIndices) { + super(docIndices, new SpatialEnvelopeVisitor.GeoPointVisitor(SpatialEnvelopeVisitor.WrapLongitude.WRAP)); + } + + @Override + protected void encodeExtent(IntBlock.Builder builder) { + // We store the 6 values as a single multi-valued field, in the same order as the fields in the Extent class + // This requires that consumers also know the meaning of the values, which they can learn from the Extent class + SpatialEnvelopeVisitor.GeoPointVisitor visitor = (SpatialEnvelopeVisitor.GeoPointVisitor) pointVisitor; + builder.beginPositionEntry(); + builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getTop())); + builder.appendInt(CoordinateEncoder.GEO.encodeY(visitor.getBottom())); + builder.appendInt(encodeLongitude(visitor.getNegLeft())); + builder.appendInt(encodeLongitude(visitor.getNegRight())); + builder.appendInt(encodeLongitude(visitor.getPosLeft())); + builder.appendInt(encodeLongitude(visitor.getPosRight())); + builder.endPositionEntry(); + } + } + + private static class TestCartesianCopier extends TestSpatialShapeExtentBlockCopier { + private TestCartesianCopier(IntVector docIndices) { + super(docIndices, new SpatialEnvelopeVisitor.CartesianPointVisitor()); + } + + @Override + protected void encodeExtent(IntBlock.Builder builder) { + // We store the 4 values as a single multi-valued field, in the same order as the fields in the Rectangle class + // This requires that consumers also know the meaning of the values, which they can learn from the Rectangle class + SpatialEnvelopeVisitor.CartesianPointVisitor visitor = (SpatialEnvelopeVisitor.CartesianPointVisitor) pointVisitor; + builder.beginPositionEntry(); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeX(visitor.getMinX())); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeX(visitor.getMaxX())); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeY(visitor.getMaxY())); + builder.appendInt(CoordinateEncoder.CARTESIAN.encodeY(visitor.getMinY())); + builder.endPositionEntry(); + } + } + } + + private static Block.Builder blockBuilder( + DataType dataType, + FieldExtractPreference extractPreference, + int estimatedSize, + BlockFactory blockFactory + ) { ElementType elementType = switch (dataType) { case SHORT -> ElementType.INT; case FLOAT, HALF_FLOAT, SCALED_FLOAT -> ElementType.DOUBLE; default -> PlannerUtils.toElementType(dataType); }; - return elementType.newBlockBuilder(estimatedSize, blockFactory); + if (extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType)) { + return blockFactory.newLongBlockBuilder(estimatedSize); + } else if (extractPreference == EXTRACT_SPATIAL_BOUNDS && DataType.isSpatial(dataType)) { + return blockFactory.newIntBlockBuilder(estimatedSize); + } else { + return elementType.newBlockBuilder(estimatedSize, blockFactory); + } + } + + private static TestBlockCopier blockCopier(DataType dataType, FieldExtractPreference extractPreference, IntVector docIndices) { + if (extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType)) { + return TestSpatialPointStatsBlockCopier.create(docIndices, dataType); + } else if (extractPreference == EXTRACT_SPATIAL_BOUNDS && DataType.isSpatial(dataType)) { + return TestSpatialShapeExtentBlockCopier.create(docIndices, dataType); + } else { + return new TestBlockCopier(docIndices); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java index f3b1d84e507a5..e58824290c49e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java @@ -155,11 +155,14 @@ protected ClusterComputeRequest mutateInstance(ClusterComputeRequest in) throws public void testFallbackIndicesOptions() throws Exception { ClusterComputeRequest request = createTestInstance(); - var version = TransportVersionUtils.randomVersionBetween(random(), TransportVersions.V_8_14_0, TransportVersions.V_8_16_0); - ClusterComputeRequest cloned = copyInstance(request, version); + var oldVersion = TransportVersionUtils.randomVersionBetween( + random(), + TransportVersions.V_8_14_0, + TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_16_0) + ); + ClusterComputeRequest cloned = copyInstance(request, oldVersion); assertThat(cloned.clusterAlias(), equalTo(request.clusterAlias())); assertThat(cloned.sessionId(), equalTo(request.sessionId())); - assertThat(cloned.configuration(), equalTo(request.configuration())); RemoteClusterPlan plan = cloned.remoteClusterPlan(); assertThat(plan.plan(), equalTo(request.remoteClusterPlan().plan())); assertThat(plan.targetIndices(), equalTo(request.remoteClusterPlan().targetIndices())); diff --git a/x-pack/plugin/fleet/build.gradle b/x-pack/plugin/fleet/build.gradle index 013d0acb4123b..4b19ff849fad8 100644 --- a/x-pack/plugin/fleet/build.gradle +++ b/x-pack/plugin/fleet/build.gradle @@ -7,7 +7,9 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-test-artifact' esplugin { name = 'x-pack-fleet' @@ -22,12 +24,7 @@ dependencies { javaRestTestImplementation(project(path: xpackModule('core'))) javaRestTestImplementation(testArtifact(project(xpackModule('core')))) compileOnly project(path: xpackModule('ilm')) -} -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.autoconfiguration.enabled', 'false' - user username: 'x_pack_rest_user', password: 'x-pack-test-password' - systemProperty 'es.queryable_built_in_roles_enabled', 'false' + clusterModules project(xpackModule('ilm')) + clusterModules project(':modules:data-streams') } diff --git a/x-pack/plugin/fleet/qa/build.gradle b/x-pack/plugin/fleet/qa/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/fleet/qa/rest/build.gradle b/x-pack/plugin/fleet/qa/rest/build.gradle deleted file mode 100644 index fda9251c7ef34..0000000000000 --- a/x-pack/plugin/fleet/qa/rest/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' - -dependencies { - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -restResources { - restApi { - include '_common', 'bulk', 'cluster', 'nodes', 'indices', 'index', 'xpack', 'security', 'fleet' - } -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - extraConfigFile 'roles.yml', file('roles.yml') - user username: 'elastic_admin', password: 'admin-password' - user username: 'fleet_unprivileged_secrets', password: 'password', role: 'unprivileged_secrets' -} diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java new file mode 100644 index 0000000000000..a835b9caa7a65 --- /dev/null +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.fleet; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +public abstract class AbstractFleetIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("x-pack-fleet") + .module("x-pack-ilm") + .module("data-streams") + .setting("xpack.security.enabled", "true") + .setting("xpack.security.autoconfiguration.enabled", "false") + .user("x_pack_rest_user", "x-pack-test-password") + .systemProperty("es.queryable_built_in_roles_enabled", "false") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java index c32b2119f0056..865c0b4af9d8a 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.test.rest.ESRestTestCase; import java.util.List; @@ -26,7 +25,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; -public class FleetDataStreamIT extends ESRestTestCase { +public class FleetDataStreamIT extends AbstractFleetIT { static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( "x_pack_rest_user", diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSecretsSystemIndexIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSecretsSystemIndexIT.java index 2a90bad72de6b..eef062b8dd173 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSecretsSystemIndexIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSecretsSystemIndexIT.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -26,7 +25,7 @@ import static org.hamcrest.Matchers.is; -public class FleetSecretsSystemIndexIT extends ESRestTestCase { +public class FleetSecretsSystemIndexIT extends AbstractFleetIT { static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( "x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 62985eb328f51..99f04d2997608 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentType; import java.util.Map; @@ -23,7 +22,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; -public class FleetSystemIndicesIT extends ESRestTestCase { +public class FleetSystemIndicesIT extends AbstractFleetIT { static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( "x_pack_rest_user", diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 63e93c5c31032..eb44107c89c2b 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -288,6 +288,7 @@ private static SystemDataStreamDescriptor fleetActionsResultsDescriptor() { composableIndexTemplate, Map.of(), ALLOWED_PRODUCTS, + FLEET_ORIGIN, ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS ); } catch (IOException e) { diff --git a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java b/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java similarity index 62% rename from x-pack/plugin/fleet/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java rename to x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java index 202149abf11e1..4b690fdc55809 100644 --- a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java +++ b/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java @@ -12,8 +12,11 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class FleetRestIT extends ESClientYamlSuiteTestCase { @@ -21,6 +24,23 @@ public FleetRestIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("x-pack-fleet") + .module("x-pack-ilm") + .module("data-streams") + .setting("xpack.license.self_generated.type", "basic") + .setting("xpack.security.enabled", "true") + .rolesFile(Resource.fromClasspath("roles.yml")) + .user("elastic_admin", "admin-password", "superuser", true) + .user("fleet_unprivileged_secrets", "password", "unprivileged_secrets", true) + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + @Override protected Settings restClientSettings() { String authentication = basicAuthHeaderValue("elastic_admin", new SecureString("admin-password".toCharArray())); diff --git a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/10_global_checkpoints.yml b/x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/10_global_checkpoints.yml similarity index 100% rename from x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/10_global_checkpoints.yml rename to x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/10_global_checkpoints.yml diff --git a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/20_wait_for_checkpoints.yml b/x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/20_wait_for_checkpoints.yml similarity index 100% rename from x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/20_wait_for_checkpoints.yml rename to x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/20_wait_for_checkpoints.yml diff --git a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/30_secrets_post.yml b/x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/30_secrets_post.yml similarity index 100% rename from x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/30_secrets_post.yml rename to x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/30_secrets_post.yml diff --git a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/40_secrets_get.yml b/x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/40_secrets_get.yml similarity index 100% rename from x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/40_secrets_get.yml rename to x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/40_secrets_get.yml diff --git a/x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/50_secrets_delete.yml b/x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/50_secrets_delete.yml similarity index 100% rename from x-pack/plugin/fleet/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/fleet/50_secrets_delete.yml rename to x-pack/plugin/fleet/src/yamlRestTest/resources/rest-api-spec/test/fleet/50_secrets_delete.yml diff --git a/x-pack/plugin/fleet/qa/rest/roles.yml b/x-pack/plugin/fleet/src/yamlRestTest/resources/roles.yml similarity index 100% rename from x-pack/plugin/fleet/qa/rest/roles.yml rename to x-pack/plugin/fleet/src/yamlRestTest/resources/roles.yml diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml index ee6094d2ffef2..c5e32cf27b1e8 100644 --- a/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/entitlement-policy.yaml @@ -3,3 +3,7 @@ ALL-UNNAMED: - write_system_properties: properties: - org.apache.xml.security.ignoreLineBreaks + - files: + - relative_path: "" + relative_to: config + mode: read diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index 256225c5ef3bf..eede706b5d403 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -6,7 +6,6 @@ apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' dependencies { - testImplementation project(':x-pack:plugin:ccr:qa') testImplementation project(':x-pack:plugin:core') testImplementation project(':x-pack:plugin:ilm') } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java index a5d966873dda1..66bcb1b201cc2 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.ccr.ESCCRRestTestCase; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.Phase; diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/ESCCRRestTestCase.java similarity index 98% rename from x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java rename to x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/ESCCRRestTestCase.java index 6701a576d6d09..47ad640bc906f 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/ESCCRRestTestCase.java @@ -4,7 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.ccr; + +package org.elasticsearch.xpack.ilm; + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ import org.apache.http.HttpHost; import org.apache.http.util.EntityUtils; @@ -40,7 +48,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; -public class ESCCRRestTestCase extends ESRestTestCase { +public abstract class ESCCRRestTestCase extends ESRestTestCase { protected final String targetCluster = System.getProperty("tests.target_cluster"); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index a1c7ebc2d8b2c..11edfff9ee108 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -455,6 +455,9 @@ public static String waitAndGetShrinkIndexName(RestClient client, String origina "GET", SHRUNKEN_INDEX_PREFIX + "*" + originalIndex + "," + originalIndex + "/_ilm/explain" ); + // Sometimes, the original index might already have been deleted, so we need to ignore unavailable (concrete) indices. + explainRequest.addParameter("ignore_unavailable", Boolean.toString(true)); + explainRequest.addParameter("expand_wildcards", "open,hidden"); explainRequest.addParameter("only_errors", Boolean.toString(false)); explainRequest.addParameter("only_managed", Boolean.toString(false)); Response response = client.performRequest(explainRequest); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/LifecycleLicenseIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/LifecycleLicenseIT.java index afaae559c1644..fe00f060d5ade 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/LifecycleLicenseIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/LifecycleLicenseIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -30,6 +29,7 @@ import org.junit.Before; import java.io.IOException; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -41,6 +41,7 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @@ -107,7 +108,9 @@ public void testSearchableSnapshotActionErrorsOnInvalidLicense() throws Exceptio // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index rolloverMaxOneDocCondition(client(), dataStream); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + List backingIndices = getDataStreamBackingIndexNames(dataStream); + assertThat(backingIndices.size(), equalTo(2)); + String backingIndexName = backingIndices.get(0); // the searchable_snapshot action should start failing (and retrying) due to invalid license assertBusy(() -> { Map explainIndex = explainIndex(client(), backingIndexName); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 28f97adec8814..31ccd61b01c18 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -9,7 +9,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.xcontent.XContentHelper; @@ -40,7 +39,6 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; -import static org.elasticsearch.xpack.TimeSeriesRestDriver.getBackingIndices; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getTemplate; @@ -80,20 +78,15 @@ public void testRolloverAction() throws Exception { indexDocument(client(), dataStream, true); - assertBusy(() -> assertTrue(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2)))); - assertBusy( - () -> assertTrue( - Boolean.parseBoolean( - (String) getIndexSettingsAsMap(DataStream.getDefaultBackingIndexName(dataStream, 2)).get("index.hidden") - ) - ) - ); - assertBusy( - () -> assertThat( - getStepKeyForIndex(client(), DataStream.getDefaultBackingIndexName(dataStream, 1)), - equalTo(PhaseCompleteStep.finalStep("hot").getKey()) - ) - ); + assertBusy(() -> { + final var backingIndices = getDataStreamBackingIndexNames(dataStream); + assertEquals(2, backingIndices.size()); + assertTrue(Boolean.parseBoolean((String) getIndexSettingsAsMap(backingIndices.get(1)).get("index.hidden"))); + }); + assertBusy(() -> { + final var backingIndices = getDataStreamBackingIndexNames(dataStream); + assertEquals(PhaseCompleteStep.finalStep("hot").getKey(), getStepKeyForIndex(client(), backingIndices.get(0))); + }); } public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { @@ -103,7 +96,7 @@ public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { indexDocument(client(), dataStream, true); - String firstGenerationIndex = DataStream.getDefaultBackingIndexName(dataStream, 1); + String firstGenerationIndex = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat(getStepKeyForIndex(client(), firstGenerationIndex).name(), equalTo(WaitForRolloverReadyStep.NAME)), 30, @@ -111,7 +104,10 @@ public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { ); rolloverMaxOneDocCondition(client(), dataStream); - assertBusy(() -> assertThat(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2)), is(true)), 30, TimeUnit.SECONDS); + assertBusy(() -> { + final var backingIndices = getDataStreamBackingIndexNames(dataStream); + assertEquals(2, backingIndices.size()); + }, 30, TimeUnit.SECONDS); // even though the first index doesn't have 2 documents to fulfill the rollover condition, it should complete the rollover action // because it's not the write index anymore @@ -122,13 +118,12 @@ public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { ); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/70595") public void testShrinkActionInPolicyWithoutHotPhase() throws Exception { createNewSingletonPolicy(client(), policyName, "warm", new ShrinkAction(1, null, false)); createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat( "original index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -142,8 +137,11 @@ public void testShrinkActionInPolicyWithoutHotPhase() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); // Wait for rollover to happen - String rolloverIndex = DataStream.getDefaultBackingIndexName(dataStream, 2); - assertBusy(() -> assertTrue("the rollover action created the rollover index", indexExists(rolloverIndex)), 30, TimeUnit.SECONDS); + assertBusy( + () -> assertEquals("the rollover action created the rollover index", 2, getDataStreamBackingIndexNames(dataStream).size()), + 30, + TimeUnit.SECONDS + ); String shrunkenIndex = waitAndGetShrinkIndexName(client(), backingIndexName); assertBusy(() -> assertTrue(indexExists(shrunkenIndex)), 30, TimeUnit.SECONDS); @@ -159,7 +157,7 @@ public void testSearchableSnapshotAction() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; assertBusy( @@ -190,7 +188,7 @@ public void testReadOnlyAction() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -220,7 +218,7 @@ public void testFreezeAction() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -249,7 +247,7 @@ public void checkForceMergeAction(String codec) throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -323,7 +321,7 @@ public void testDataStreamWithMultipleIndicesAndWriteIndexInDeletePhase() throws client().performRequest(new Request("POST", dataStream + "/_rollover")); indexDocument(client(), dataStream, true); - String secondGenerationIndex = getBackingIndices(client(), dataStream).get(1); + String secondGenerationIndex = getDataStreamBackingIndexNames(dataStream).get(1); assertBusy(() -> { Request explainRequest = new Request("GET", "/_data_stream/" + dataStream); Response response = client().performRequest(explainRequest); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 4c53d711ffdef..dddb07fae42c5 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -1221,7 +1220,7 @@ private void assertHistoryIsPresent( } // Finally, check that the history index is in a good state - String historyIndexName = DataStream.getDefaultBackingIndexName("ilm-history-7", 1); + String historyIndexName = getDataStreamBackingIndexNames("ilm-history-7").get(0); Response explainHistoryIndex = client().performRequest(new Request("GET", historyIndexName + "/_lifecycle/explain")); Map responseMap; try (InputStream is = explainHistoryIndex.getEntity().getContent()) { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index d4ecff4238591..17a1ad7ed9040 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata.DownsampleTaskStatus; import org.elasticsearch.common.Strings; @@ -49,7 +48,6 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; -import static org.elasticsearch.xpack.TimeSeriesRestDriver.getBackingIndices; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.index; @@ -318,7 +316,7 @@ public void testTsdbDataStreams() throws Exception { index(client(), dataStream, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -365,7 +363,7 @@ public void testILMWaitsForTimeSeriesEndTimeToLapse() throws Exception { String now = DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(Instant.now()); index(client(), dataStream, true, null, "@timestamp", now, "volume", 11.0, "metricset", randomAlphaOfLength(5)); - String backingIndexName = getBackingIndices(client(), dataStream).get(0); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -459,7 +457,7 @@ public void testDownsampleTwice() throws Exception { index(client(), dataStream, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - String firstBackingIndex = DataStream.getDefaultBackingIndexName(dataStream, 1); + String firstBackingIndex = getDataStreamBackingIndexNames(dataStream).get(0); logger.info("--> firstBackingIndex: {}", firstBackingIndex); assertBusy( () -> assertThat( @@ -540,7 +538,7 @@ public void testDownsampleTwiceSameInterval() throws Exception { index(client(), dataStream, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - String firstBackingIndex = getBackingIndices(client(), dataStream).get(0); + String firstBackingIndex = getDataStreamBackingIndexNames(dataStream).get(0); logger.info("--> firstBackingIndex: {}", firstBackingIndex); assertBusy( () -> assertThat( diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 5c07cbada7d39..8eae3f17ae5ef 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -12,7 +12,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.routing.allocation.DataTier; @@ -56,7 +55,6 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.createPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; -import static org.elasticsearch.xpack.TimeSeriesRestDriver.getBackingIndices; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfPrimarySegments; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; @@ -103,7 +101,9 @@ public void testSearchableSnapshotAction() throws Exception { // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index rolloverMaxOneDocCondition(client(), dataStream); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + List backingIndices = getDataStreamBackingIndexNames(dataStream); + assertThat(backingIndices.size(), equalTo(2)); + String backingIndexName = backingIndices.get(0); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; assertTrue(waitUntil(() -> { try { @@ -134,7 +134,8 @@ public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exceptio indexDocument(client(), dataStream, true); } - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + List backingIndices = getDataStreamBackingIndexNames(dataStream); + String backingIndexName = backingIndices.get(0); Integer preLifecycleBackingIndexSegments = getNumberOfPrimarySegments(client(), backingIndexName); assertThat(preLifecycleBackingIndexSegments, greaterThanOrEqualTo(1)); @@ -208,7 +209,9 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index rolloverMaxOneDocCondition(client(), dataStream); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + List backingIndices = getDataStreamBackingIndexNames(dataStream); + assertThat(backingIndices.size(), equalTo(2)); + String backingIndexName = backingIndices.get(0); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; // let's wait for ILM to finish @@ -300,7 +303,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index indexDocument(client(), dataStream, true); - var backingIndices = getBackingIndices(client(), dataStream); + var backingIndices = getDataStreamBackingIndexNames(dataStream); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndices.get(0); assertTrue(waitUntil(() -> { try { @@ -378,10 +381,8 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex // indexing only one document as we want only one rollover to be triggered indexDocument(client(), dataStream, true); - String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + DataStream.getDefaultBackingIndexName( - dataStream, - 1L - ); + String backingIndexName = getDataStreamBackingIndexNames(dataStream).get(0); + String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; assertTrue(waitUntil(() -> { try { return indexExists(searchableSnapMountedIndexName); @@ -856,7 +857,7 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception // Create the data stream. assertOK(client().performRequest(new Request("PUT", "_data_stream/" + dataStream))); - var backingIndices = getBackingIndices(client(), dataStream); + var backingIndices = getDataStreamBackingIndexNames(dataStream); String firstGenIndex = backingIndices.get(0); Map indexSettings = getIndexSettingsAsMap(firstGenIndex); assertThat(indexSettings.get(DataTier.TIER_PREFERENCE), is("data_hot")); @@ -907,7 +908,9 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index rolloverMaxOneDocCondition(client(), dataStream); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + List backingIndices = getDataStreamBackingIndexNames(dataStream); + assertThat(backingIndices.size(), equalTo(2)); + String backingIndexName = backingIndices.get(0); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; assertTrue(waitUntil(() -> { try { @@ -1011,7 +1014,9 @@ public void testSearchableSnapshotReplicateFor() throws Exception { // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index rolloverMaxOneDocCondition(client(), dataStream); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + List backingIndices = getDataStreamBackingIndexNames(dataStream); + assertThat(backingIndices.size(), equalTo(2)); + String backingIndexName = backingIndices.get(0); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; assertTrue(waitUntil(() -> { try { diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java index 2c4c1c9e20bb6..2a3080b828393 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; @@ -103,7 +102,7 @@ public void testShrinkOnTiers() throws Exception { .get(); logger.info("--> explain: {}", Strings.toString(explain)); - String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); + String backingIndexName = getDataStreamBackingIndexNames(index).get(0); IndexLifecycleExplainResponse indexResp = null; for (Map.Entry indexNameAndResp : explain.getIndexResponses().entrySet()) { if (indexNameAndResp.getKey().startsWith(SHRUNKEN_INDEX_PREFIX) && indexNameAndResp.getKey().contains(backingIndexName)) { diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java index b91a309a23ae5..592c5a099d7a5 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; @@ -103,7 +102,7 @@ public void testShrinkOnTiers() throws Exception { .get(); logger.info("--> explain: {}", Strings.toString(explain)); - String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); + String backingIndexName = getDataStreamBackingIndexNames(index).get(0); IndexLifecycleExplainResponse indexResp = null; for (Map.Entry indexNameAndResp : explain.getIndexResponses().entrySet()) { if (indexNameAndResp.getKey().startsWith(SHRUNKEN_INDEX_PREFIX) && indexNameAndResp.getKey().contains(backingIndexName)) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java index 85739dcd0dcfb..5ec48aee9dee9 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java @@ -322,7 +322,11 @@ void maybeRunAsyncAction(ClusterState currentState, IndexMetadata indexMetadata, logger.warn("current step [{}] for index [{}] with policy [{}] is not recognized", currentStepKey, index, policy); return; } - + if (expectedStepKey.phase() == null && expectedStepKey.name() == null && expectedStepKey.action() == null) { + // ILM is stopped, so do not try to run async action + logger.debug("expected step for index [{}] with policy [{}] is [{}], not running async action", index, policy, expectedStepKey); + return; + } logger.trace( "[{}] maybe running async action step ({}) with current step {}", index, diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRetryAction.java index 1000bd1e68249..6aea4fbf85091 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRetryAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRetryAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.RetryActionRequest; import java.util.List; @@ -37,7 +38,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { final var indices = Strings.splitStringByCommaToArray(restRequest.param("index")); - final var request = new TransportRetryAction.Request(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest), indices); + final var request = new RetryActionRequest(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest), indices); request.indices(indices); request.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen())); return channel -> client.execute(ILMActions.RETRY, request, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java index 38ed32a8aea88..bfc077cfd82fb 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java @@ -10,11 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -26,24 +22,18 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.RetryActionRequest; import org.elasticsearch.xpack.ilm.IndexLifecycleService; -import java.io.IOException; -import java.util.Arrays; -import java.util.Objects; - -public class TransportRetryAction extends TransportMasterNodeAction { +public class TransportRetryAction extends TransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportRetryAction.class); @@ -64,7 +54,7 @@ public TransportRetryAction( clusterService, threadPool, actionFilters, - Request::new, + RetryActionRequest::new, AcknowledgedResponse::readFrom, EsExecutors.DIRECT_EXECUTOR_SERVICE ); @@ -72,7 +62,17 @@ public TransportRetryAction( } @Override - protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { + protected void masterOperation( + Task task, + RetryActionRequest request, + ClusterState state, + ActionListener listener + ) { + if (request.requireError() == false) { + maybeRunAsyncAction(state, request.indices()); + listener.onResponse(AcknowledgedResponse.TRUE); + return; + } submitUnbatchedTask("ilm-re-run", new AckedClusterStateUpdateTask(request, listener) { @Override public ClusterState execute(ClusterState currentState) { @@ -81,101 +81,33 @@ public ClusterState execute(ClusterState currentState) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { - for (String index : request.indices()) { - IndexMetadata idxMeta = newState.metadata().index(index); - LifecycleExecutionState lifecycleState = idxMeta.getLifecycleExecutionState(); - StepKey retryStep = new StepKey(lifecycleState.phase(), lifecycleState.action(), lifecycleState.step()); - if (idxMeta == null) { - // The index has somehow been deleted - there shouldn't be any opportunity for this to happen, but just in case. - logger.debug( - "index [" - + index - + "] has been deleted after moving to step [" - + lifecycleState.step() - + "], skipping async action check" - ); - return; - } - indexLifecycleService.maybeRunAsyncAction(newState, idxMeta, retryStep); - } + maybeRunAsyncAction(newState, request.indices()); } }); } + private void maybeRunAsyncAction(ClusterState state, String[] indices) { + for (String index : indices) { + IndexMetadata idxMeta = state.metadata().index(index); + if (idxMeta == null) { + // The index has somehow been deleted - there shouldn't be any opportunity for this to happen, but just in case. + logger.debug("index [" + index + "] has been deleted, skipping async action check"); + return; + } + LifecycleExecutionState lifecycleState = idxMeta.getLifecycleExecutionState(); + StepKey retryStep = new StepKey(lifecycleState.phase(), lifecycleState.action(), lifecycleState.step()); + indexLifecycleService.maybeRunAsyncAction(state, idxMeta, retryStep); + } + } + @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { clusterService.submitUnbatchedStateUpdateTask(source, task); } @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { + protected ClusterBlockException checkBlock(RetryActionRequest request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } - public static class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { - private String[] indices; - private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen(); - - public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... indices) { - super(masterNodeTimeout, ackTimeout); - this.indices = indices; - } - - public Request(StreamInput in) throws IOException { - super(in); - this.indices = in.readStringArray(); - this.indicesOptions = IndicesOptions.readIndicesOptions(in); - } - - @Override - public Request indices(String... indices) { - this.indices = indices; - return this; - } - - @Override - public String[] indices() { - return indices; - } - - @Override - public IndicesOptions indicesOptions() { - return indicesOptions; - } - - public Request indicesOptions(IndicesOptions indicesOptions) { - this.indicesOptions = indicesOptions; - return this; - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeStringArray(indices); - indicesOptions.writeIndicesOptions(out); - } - - @Override - public int hashCode() { - return Objects.hash(Arrays.hashCode(indices), indicesOptions); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (obj.getClass() != getClass()) { - return false; - } - Request other = (Request) obj; - return Objects.deepEquals(indices, other.indices) && Objects.equals(indicesOptions, other.indicesOptions); - } - - } } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/RetryRequestTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/RetryRequestTests.java index 4f053ddc2caa4..67fcc781f1edd 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/RetryRequestTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/RetryRequestTests.java @@ -11,14 +11,15 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ilm.action.RetryActionRequest; import java.util.Arrays; -public class RetryRequestTests extends AbstractWireSerializingTestCase { +public class RetryRequestTests extends AbstractWireSerializingTestCase { @Override - protected TransportRetryAction.Request createTestInstance() { - final var request = new TransportRetryAction.Request( + protected RetryActionRequest createTestInstance() { + final var request = new RetryActionRequest( TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomBoolean() ? Strings.EMPTY_ARRAY : generateRandomStringArray(20, 20, false) @@ -36,19 +37,23 @@ protected TransportRetryAction.Request createTestInstance() { ); request.indicesOptions(indicesOptions); } + if (randomBoolean()) { + request.requireError(randomBoolean()); + } return request; } @Override - protected Writeable.Reader instanceReader() { - return TransportRetryAction.Request::new; + protected Writeable.Reader instanceReader() { + return RetryActionRequest::new; } @Override - protected TransportRetryAction.Request mutateInstance(TransportRetryAction.Request instance) { + protected RetryActionRequest mutateInstance(RetryActionRequest instance) { String[] indices = instance.indices(); IndicesOptions indicesOptions = instance.indicesOptions(); - switch (between(0, 1)) { + boolean requireError = instance.requireError(); + switch (between(0, 2)) { case 0 -> indices = randomValueOtherThanMany( i -> Arrays.equals(i, instance.indices()), () -> generateRandomStringArray(20, 10, false, true) @@ -66,10 +71,12 @@ protected TransportRetryAction.Request mutateInstance(TransportRetryAction.Reque randomBoolean() ) ); + case 2 -> requireError = requireError == false; default -> throw new AssertionError("Illegal randomisation branch"); } - final var newRequest = new TransportRetryAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indices); + final var newRequest = new RetryActionRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indices); newRequest.indicesOptions(indicesOptions); + newRequest.requireError(requireError); return newRequest; } } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java index d0f797e9f8fab..81c1a8dc7a5ba 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java @@ -39,6 +39,10 @@ public class BaseMockEISAuthServerTest extends ESRestTestCase { .setting("xpack.security.enabled", "true") // Adding both settings unless one feature flag is disabled in a particular environment .setting("xpack.inference.elastic.url", mockEISServer::getUrl) + // If we don't disable this there's a very small chance that the authorization code could attempt to make two + // calls which would result in a test failure because the webserver is only expecting a single request + // So to ensure we avoid that all together, this flag indicates that we'll only perform a single authorization request + .setting("xpack.inference.elastic.periodic_authorization_enabled", "false") // This plugin is located in the inference/qa/test-service-plugin package, look for TestInferenceServicePlugin .plugin("inference-service-test") .user("x_pack_rest_user", "x-pack-test-password") diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java index 0a2200ff912ac..47f34fa486daf 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CreateFromDeploymentIT.java @@ -40,9 +40,35 @@ public void testAttachToDeployment() throws IOException { is(Map.of("num_allocations", 1, "num_threads", 1, "model_id", "attach_to_deployment", "deployment_id", "existing_deployment")) ); + var getModel = getModel(inferenceId); + serviceSettings = getModel.get("service_settings"); + assertThat( + getModel.toString(), + serviceSettings, + is(Map.of("num_allocations", 1, "num_threads", 1, "model_id", "attach_to_deployment", "deployment_id", "existing_deployment")) + ); + var results = infer(inferenceId, List.of("washing machine")); assertNotNull(results.get("sparse_embedding")); + var updatedNumAllocations = randomIntBetween(1, 10); + var updatedEndpointConfig = updateEndpoint(inferenceId, updatedEndpointConfig(updatedNumAllocations), TaskType.SPARSE_EMBEDDING); + assertThat( + updatedEndpointConfig.get("service_settings"), + is( + Map.of( + "num_allocations", + updatedNumAllocations, + "num_threads", + 1, + "model_id", + "attach_to_deployment", + "deployment_id", + "existing_deployment" + ) + ) + ); + deleteModel(inferenceId); // assert deployment not stopped var stats = (List>) getTrainedModelStats(modelId).get("trained_model_stats"); @@ -80,9 +106,46 @@ public void testAttachWithModelId() throws IOException { ) ); + var getModel = getModel(inferenceId); + serviceSettings = getModel.get("service_settings"); + assertThat( + getModel.toString(), + serviceSettings, + is( + Map.of( + "num_allocations", + 1, + "num_threads", + 1, + "model_id", + "attach_with_model_id", + "deployment_id", + "existing_deployment_with_model_id" + ) + ) + ); + var results = infer(inferenceId, List.of("washing machine")); assertNotNull(results.get("sparse_embedding")); + var updatedNumAllocations = randomIntBetween(1, 10); + var updatedEndpointConfig = updateEndpoint(inferenceId, updatedEndpointConfig(updatedNumAllocations), TaskType.SPARSE_EMBEDDING); + assertThat( + updatedEndpointConfig.get("service_settings"), + is( + Map.of( + "num_allocations", + updatedNumAllocations, + "num_threads", + 1, + "model_id", + "attach_with_model_id", + "deployment_id", + "existing_deployment_with_model_id" + ) + ) + ); + stopMlNodeDeployment(deploymentId); } @@ -189,6 +252,16 @@ private String endpointConfig(String modelId, String deploymentId) { """, modelId, deploymentId); } + private String updatedEndpointConfig(int numAllocations) { + return Strings.format(""" + { + "service_settings": { + "num_allocations": %d + } + } + """, numAllocations); + } + private Response startMlNodeDeploymemnt(String modelId, String deploymentId) throws IOException { String endPoint = "/_ml/trained_models/" + modelId diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 546eab471a077..36a4b95a7ca23 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -238,6 +238,11 @@ static Map updateEndpoint(String inferenceID, String modelConfig return putRequest(endpoint, modelConfig); } + static Map updateEndpoint(String inferenceID, String modelConfig) throws IOException { + String endpoint = Strings.format("_inference/%s/_update", inferenceID); + return putRequest(endpoint, modelConfig); + } + protected Map putPipeline(String pipelineId, String modelId) throws IOException { String endpoint = Strings.format("_ingest/pipeline/%s", pipelineId); String body = """ @@ -355,7 +360,7 @@ protected Deque unifiedCompletionInferOnMockService( List input, @Nullable Consumer responseConsumerCallback ) throws Exception { - var endpoint = Strings.format("_inference/%s/%s/_unified", taskType, modelId); + var endpoint = Strings.format("_inference/%s/%s/_stream", taskType, modelId); return callAsyncUnified(endpoint, input, "user", responseConsumerCallback); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index b786cd1298495..793b3f7a9a349 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -369,6 +369,61 @@ public void testUnifiedCompletionInference() throws Exception { } } + public void testUpdateEndpointWithWrongTaskTypeInURL() throws IOException { + putModel("sparse_embedding_model", mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); + var e = expectThrows( + ResponseException.class, + () -> updateEndpoint( + "sparse_embedding_model", + updateConfig(null, randomAlphaOfLength(10), randomIntBetween(1, 10)), + TaskType.TEXT_EMBEDDING + ) + ); + assertThat(e.getMessage(), containsString("Task type must match the task type of the existing endpoint")); + } + + public void testUpdateEndpointWithWrongTaskTypeInBody() throws IOException { + putModel("sparse_embedding_model", mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); + var e = expectThrows( + ResponseException.class, + () -> updateEndpoint( + "sparse_embedding_model", + updateConfig(TaskType.TEXT_EMBEDDING, randomAlphaOfLength(10), randomIntBetween(1, 10)) + ) + ); + assertThat(e.getMessage(), containsString("Task type must match the task type of the existing endpoint")); + } + + public void testUpdateEndpointWithTaskTypeInURL() throws IOException { + testUpdateEndpoint(false, true); + } + + public void testUpdateEndpointWithTaskTypeInBody() throws IOException { + testUpdateEndpoint(true, false); + } + + public void testUpdateEndpointWithTaskTypeInBodyAndURL() throws IOException { + testUpdateEndpoint(true, true); + } + + @SuppressWarnings("unchecked") + private void testUpdateEndpoint(boolean taskTypeInBody, boolean taskTypeInURL) throws IOException { + String endpointId = "sparse_embedding_model"; + putModel(endpointId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); + + int temperature = randomIntBetween(1, 10); + var expectedConfig = updateConfig(taskTypeInBody ? TaskType.SPARSE_EMBEDDING : null, randomAlphaOfLength(1), temperature); + Map updatedEndpoint; + if (taskTypeInURL) { + updatedEndpoint = updateEndpoint(endpointId, expectedConfig, TaskType.SPARSE_EMBEDDING); + } else { + updatedEndpoint = updateEndpoint(endpointId, expectedConfig); + } + + Map updatedTaskSettings = (Map) updatedEndpoint.get("task_settings"); + assertEquals(temperature, updatedTaskSettings.get("temperature")); + } + private static Iterator expectedResultsIterator(List input) { // The Locale needs to be ROOT to match what the test service is going to respond with return Stream.concat(input.stream().map(s -> s.toUpperCase(Locale.ROOT)).map(InferenceCrudIT::expectedResult), Stream.of("[DONE]")) diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterBasicLicenseIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterBasicLicenseIT.java new file mode 100644 index 0000000000000..4fc97662166f0 --- /dev/null +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterBasicLicenseIT.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action.filter; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; +import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.license.LicenseSettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; +import org.elasticsearch.xpack.inference.Utils; +import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension; +import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticTextInput; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; + +public class ShardBulkInferenceActionFilterBasicLicenseIT extends ESIntegTestCase { + public static final String INDEX_NAME = "test-index"; + + private final boolean useLegacyFormat; + + public ShardBulkInferenceActionFilterBasicLicenseIT(boolean useLegacyFormat) { + this.useLegacyFormat = useLegacyFormat; + } + + @ParametersFactory + public static Iterable parameters() { + return List.of(new Object[] { true }, new Object[] { false }); + } + + @Before + public void setup() throws Exception { + Utils.storeSparseModel(client()); + Utils.storeDenseModel( + client(), + randomIntBetween(1, 100), + // dot product means that we need normalized vectors; it's not worth doing that in this test + randomValueOtherThan(SimilarityMeasure.DOT_PRODUCT, () -> randomFrom(SimilarityMeasure.values())), + // TODO: Allow element type BIT once TestDenseInferenceServiceExtension supports it + randomValueOtherThan(DenseVectorFieldMapper.ElementType.BIT, () -> randomFrom(DenseVectorFieldMapper.ElementType.values())) + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "basic").build(); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LocalStateInferencePlugin.class); + } + + @Override + public Settings indexSettings() { + var builder = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat); + return builder.build(); + } + + public void testLicenseInvalidForInference() { + prepareCreate(INDEX_NAME).setMapping( + String.format( + Locale.ROOT, + """ + { + "properties": { + "sparse_field": { + "type": "semantic_text", + "inference_id": "%s" + }, + "dense_field": { + "type": "semantic_text", + "inference_id": "%s" + } + } + } + """, + TestSparseInferenceServiceExtension.TestInferenceService.NAME, + TestDenseInferenceServiceExtension.TestInferenceService.NAME + ) + ).get(); + + BulkRequestBuilder bulkRequest = client().prepareBulk(); + int totalBulkReqs = randomIntBetween(2, 100); + for (int i = 0; i < totalBulkReqs; i++) { + Map source = new HashMap<>(); + source.put("sparse_field", randomSemanticTextInput()); + source.put("dense_field", randomSemanticTextInput()); + + bulkRequest.add(new IndexRequestBuilder(client()).setIndex(INDEX_NAME).setId(Long.toString(i)).setSource(source)); + } + + BulkResponse bulkResponse = bulkRequest.get(); + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + assertTrue(bulkItemResponse.isFailed()); + assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(ElasticsearchSecurityException.class)); + assertThat( + bulkItemResponse.getFailure().getCause().getMessage(), + containsString(Strings.format("current license is non-compliant for [%s]", XPackField.INFERENCE)) + ); + } + } + + public void testNullSourceSucceeds() { + prepareCreate(INDEX_NAME).setMapping( + String.format( + Locale.ROOT, + """ + { + "properties": { + "sparse_field": { + "type": "semantic_text", + "inference_id": "%s" + }, + "dense_field": { + "type": "semantic_text", + "inference_id": "%s" + } + } + } + """, + TestSparseInferenceServiceExtension.TestInferenceService.NAME, + TestDenseInferenceServiceExtension.TestInferenceService.NAME + ) + ).get(); + + BulkRequestBuilder bulkRequest = client().prepareBulk(); + int totalBulkReqs = randomIntBetween(2, 100); + Map source = new HashMap<>(); + source.put("sparse_field", null); + source.put("dense_field", null); + for (int i = 0; i < totalBulkReqs; i++) { + bulkRequest.add(new IndexRequestBuilder(client()).setIndex(INDEX_NAME).setId(Long.toString(i)).setSource(source)); + } + + BulkResponse bulkResponse = bulkRequest.get(); + assertFalse(bulkResponse.hasFailures()); + } +} diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java index 8e61323d3ca94..35c10294fb251 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -44,6 +45,7 @@ import java.util.Set; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticTextInput; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class ShardBulkInferenceActionFilterIT extends ESIntegTestCase { @@ -80,6 +82,11 @@ public void setup() throws Exception { ); } + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(LocalStateInferencePlugin.class); @@ -183,4 +190,48 @@ public void testBulkOperations() throws Exception { searchResponse.decRef(); } } + + public void testItemFailures() { + prepareCreate(INDEX_NAME).setMapping( + String.format( + Locale.ROOT, + """ + { + "properties": { + "sparse_field": { + "type": "semantic_text", + "inference_id": "%s" + }, + "dense_field": { + "type": "semantic_text", + "inference_id": "%s" + } + } + } + """, + TestSparseInferenceServiceExtension.TestInferenceService.NAME, + TestDenseInferenceServiceExtension.TestInferenceService.NAME + ) + ).get(); + + BulkRequestBuilder bulkReqBuilder = client().prepareBulk(); + int totalBulkSize = randomIntBetween(100, 200); // Use a bulk request size large enough to require batching + for (int bulkSize = 0; bulkSize < totalBulkSize; bulkSize++) { + String id = Integer.toString(bulkSize); + + // Set field values that will cause errors when generating inference requests + Map source = new HashMap<>(); + source.put("sparse_field", List.of(Map.of("foo", "bar"), Map.of("baz", "bar"))); + source.put("dense_field", List.of(Map.of("foo", "bar"), Map.of("baz", "bar"))); + + bulkReqBuilder.add(new IndexRequestBuilder(client()).setIndex(INDEX_NAME).setId(id).setSource(source)); + } + + BulkResponse bulkResponse = bulkReqBuilder.get(); + assertThat(bulkResponse.hasFailures(), equalTo(true)); + for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { + assertThat(bulkItemResponse.isFailed(), equalTo(true)); + assertThat(bulkItemResponse.getFailureMessage(), containsString("expected [String|Number|Boolean]")); + } + } } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java new file mode 100644 index 0000000000000..cc1556d414215 --- /dev/null +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java @@ -0,0 +1,280 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.integration; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.MinimalServiceSettings; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationRequestHandler; +import org.junit.After; +import org.junit.Before; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.hamcrest.CoreMatchers.is; +import static org.mockito.Mockito.mock; + +public class InferenceRevokeDefaultEndpointsIT extends ESSingleNodeTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + + private ModelRegistry modelRegistry; + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private String gatewayUrl; + + @Before + public void createComponents() throws Exception { + threadPool = createThreadPool(inferenceUtilityPool()); + webServer.start(); + gatewayUrl = getUrl(webServer); + modelRegistry = new ModelRegistry(client()); + } + + @After + public void shutdown() { + terminate(threadPool); + webServer.close(); + } + + @Override + protected boolean resetNodeAfterTest() { + return true; + } + + @Override + protected Collection> getPlugins() { + return pluginList(ReindexPlugin.class); + } + + public void testDefaultConfigs_Returns_DefaultChatCompletion_V1_WhenTaskTypeIsCorrect() throws Exception { + String responseJson = """ + { + "models": [ + { + "model_name": "rainbow-sprinkles", + "task_types": ["chat"] + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + try (var service = createElasticInferenceService()) { + ensureAuthorizationCallFinished(service); + assertThat(service.supportedStreamingTasks(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.ANY))); + assertThat( + service.defaultConfigIds(), + is( + List.of( + new InferenceService.DefaultConfigId(".rainbow-sprinkles-elastic", MinimalServiceSettings.chatCompletion(), service) + ) + ) + ); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.CHAT_COMPLETION))); + + PlainActionFuture> listener = new PlainActionFuture<>(); + service.defaultConfigs(listener); + assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); + } + } + + public void testRemoves_DefaultChatCompletion_V1_WhenAuthorizationReturnsEmpty() throws Exception { + { + String responseJson = """ + { + "models": [ + { + "model_name": "rainbow-sprinkles", + "task_types": ["chat"] + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + try (var service = createElasticInferenceService()) { + ensureAuthorizationCallFinished(service); + + assertThat(service.supportedStreamingTasks(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.ANY))); + assertThat( + service.defaultConfigIds(), + is( + List.of( + new InferenceService.DefaultConfigId( + ".rainbow-sprinkles-elastic", + MinimalServiceSettings.chatCompletion(), + service + ) + ) + ) + ); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.CHAT_COMPLETION))); + + PlainActionFuture> listener = new PlainActionFuture<>(); + service.defaultConfigs(listener); + assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); + + var getModelListener = new PlainActionFuture(); + // persists the default endpoints + modelRegistry.getModel(".rainbow-sprinkles-elastic", getModelListener); + + var inferenceEntity = getModelListener.actionGet(TIMEOUT); + assertThat(inferenceEntity.inferenceEntityId(), is(".rainbow-sprinkles-elastic")); + assertThat(inferenceEntity.taskType(), is(TaskType.CHAT_COMPLETION)); + } + } + { + String noAuthorizationResponseJson = """ + { + "models": [] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(noAuthorizationResponseJson)); + + try (var service = createElasticInferenceService()) { + ensureAuthorizationCallFinished(service); + + assertThat(service.supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class))); + assertTrue(service.defaultConfigIds().isEmpty()); + assertThat(service.supportedTaskTypes(), is(EnumSet.noneOf(TaskType.class))); + + var getModelListener = new PlainActionFuture(); + modelRegistry.getModel(".rainbow-sprinkles-elastic", getModelListener); + + var exception = expectThrows(ResourceNotFoundException.class, () -> getModelListener.actionGet(TIMEOUT)); + assertThat(exception.getMessage(), is("Inference endpoint not found [.rainbow-sprinkles-elastic]")); + } + } + } + + public void testRemoves_DefaultChatCompletion_V1_WhenAuthorizationDoesNotReturnAuthForIt() throws Exception { + { + String responseJson = """ + { + "models": [ + { + "model_name": "rainbow-sprinkles", + "task_types": ["chat"] + }, + { + "model_name": "elser-v2", + "task_types": ["embed/text/sparse"] + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + try (var service = createElasticInferenceService()) { + ensureAuthorizationCallFinished(service); + + assertThat(service.supportedStreamingTasks(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.ANY))); + assertThat( + service.defaultConfigIds(), + is( + List.of( + new InferenceService.DefaultConfigId( + ".rainbow-sprinkles-elastic", + MinimalServiceSettings.chatCompletion(), + service + ) + ) + ) + ); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.SPARSE_EMBEDDING))); + + PlainActionFuture> listener = new PlainActionFuture<>(); + service.defaultConfigs(listener); + assertThat(listener.actionGet(TIMEOUT).get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); + + var getModelListener = new PlainActionFuture(); + // persists the default endpoints + modelRegistry.getModel(".rainbow-sprinkles-elastic", getModelListener); + + var inferenceEntity = getModelListener.actionGet(TIMEOUT); + assertThat(inferenceEntity.inferenceEntityId(), is(".rainbow-sprinkles-elastic")); + assertThat(inferenceEntity.taskType(), is(TaskType.CHAT_COMPLETION)); + } + } + { + String noAuthorizationResponseJson = """ + { + "models": [ + { + "model_name": "elser-v2", + "task_types": ["embed/text/sparse"] + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(noAuthorizationResponseJson)); + + try (var service = createElasticInferenceService()) { + ensureAuthorizationCallFinished(service); + + assertThat(service.supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class))); + assertTrue(service.defaultConfigIds().isEmpty()); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING))); + + var getModelListener = new PlainActionFuture(); + modelRegistry.getModel(".rainbow-sprinkles-elastic", getModelListener); + + var exception = expectThrows(ResourceNotFoundException.class, () -> getModelListener.actionGet(TIMEOUT)); + assertThat(exception.getMessage(), is("Inference endpoint not found [.rainbow-sprinkles-elastic]")); + } + } + } + + private void ensureAuthorizationCallFinished(ElasticInferenceService service) { + service.onNodeStarted(); + service.waitForFirstAuthorizationToComplete(TIMEOUT); + } + + private ElasticInferenceService createElasticInferenceService() { + var httpManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, httpManager); + + return new ElasticInferenceService( + senderFactory, + createWithEmptySettings(threadPool), + ElasticInferenceServiceSettingsTests.create(gatewayUrl), + modelRegistry, + new ElasticInferenceServiceAuthorizationRequestHandler(gatewayUrl, threadPool) + ); + } +} diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index dfdca6226efd3..8f6e9f8cb5f21 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -10,10 +10,12 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.InferenceService; @@ -51,7 +53,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; @@ -70,6 +74,7 @@ import static org.mockito.Mockito.mock; public class ModelRegistryIT extends ESSingleNodeTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); private ModelRegistry modelRegistry; @@ -195,6 +200,56 @@ public void testDeleteModel() throws Exception { assertThat(exceptionHolder.get().getMessage(), containsString("Inference endpoint not found [model1]")); } + public void testNonExistentDeleteModel_DoesNotThrowAnException() { + var listener = new PlainActionFuture(); + + modelRegistry.deleteModel("non-existent-model", listener); + assertTrue(listener.actionGet(TIMEOUT)); + } + + public void testRemoveDefaultConfigs_DoesNotThrowAnException_WhenSearchingForNonExistentInferenceEndpointIds() { + var listener = new PlainActionFuture(); + + modelRegistry.deleteModels(Set.of("non-existent-model", "abc"), listener); + assertTrue(listener.actionGet(TIMEOUT)); + } + + public void testRemoveDefaultConfigs_RemovesModelsFromPersistentStorage_AndInMemoryCache() { + var service = mock(InferenceService.class); + + var defaultConfigs = new ArrayList(); + var defaultIds = new ArrayList(); + for (var id : new String[] { "model1", "model2", "model3" }) { + var modelSettings = ModelRegistryTests.randomMinimalServiceSettings(); + defaultConfigs.add(createModel(id, modelSettings.taskType(), "name")); + defaultIds.add(new InferenceService.DefaultConfigId(id, modelSettings, service)); + } + + doAnswer(invocation -> { + ActionListener> listener = invocation.getArgument(0); + listener.onResponse(defaultConfigs); + return Void.TYPE; + }).when(service).defaultConfigs(any()); + + defaultIds.forEach(modelRegistry::addDefaultIds); + + var getModelsListener = new PlainActionFuture>(); + modelRegistry.getAllModels(true, getModelsListener); + var unparsedModels = getModelsListener.actionGet(TIMEOUT); + assertThat(unparsedModels.size(), is(3)); + + var removeModelsListener = new PlainActionFuture(); + + modelRegistry.removeDefaultConfigs(Set.of("model1", "model2", "model3"), removeModelsListener); + assertTrue(removeModelsListener.actionGet(TIMEOUT)); + + var getModelsAfterDeleteListener = new PlainActionFuture>(); + // the models should have been removed from the in memory cache, if not they they will be persisted again by this call + modelRegistry.getAllModels(true, getModelsAfterDeleteListener); + var unparsedModelsAfterDelete = getModelsAfterDeleteListener.actionGet(TIMEOUT); + assertThat(unparsedModelsAfterDelete.size(), is(0)); + } + public void testGetModelsByTaskType() throws InterruptedException { var service = "foo"; var sparseAndTextEmbeddingModels = new ArrayList(); @@ -315,8 +370,7 @@ public void testGetAllModels_WithDefaults() throws Exception { } doAnswer(invocation -> { - @SuppressWarnings("unchecked") - var listener = (ActionListener>) invocation.getArguments()[0]; + ActionListener> listener = invocation.getArgument(0); listener.onResponse(defaultConfigs); return Void.TYPE; }).when(service).defaultConfigs(any()); @@ -381,8 +435,7 @@ public void testGetAllModels_OnlyDefaults() throws Exception { } doAnswer(invocation -> { - @SuppressWarnings("unchecked") - var listener = (ActionListener>) invocation.getArguments()[0]; + ActionListener> listener = invocation.getArgument(0); listener.onResponse(defaultConfigs); return Void.TYPE; }).when(service).defaultConfigs(any()); @@ -424,8 +477,7 @@ public void testGetAllModels_withDoNotPersist() throws Exception { } doAnswer(invocation -> { - @SuppressWarnings("unchecked") - var listener = (ActionListener>) invocation.getArguments()[0]; + ActionListener> listener = invocation.getArgument(0); listener.onResponse(defaultConfigs); return Void.TYPE; }).when(service).defaultConfigs(any()); @@ -466,8 +518,7 @@ public void testGet_WithDefaults() throws InterruptedException { ); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - var listener = (ActionListener>) invocation.getArguments()[0]; + ActionListener> listener = invocation.getArgument(0); listener.onResponse(defaultConfigs); return Void.TYPE; }).when(service).defaultConfigs(any()); @@ -520,8 +571,7 @@ public void testGetByTaskType_WithDefaults() throws Exception { defaultIds.add(new InferenceService.DefaultConfigId("default-chat", MinimalServiceSettings.completion(), service)); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - var listener = (ActionListener>) invocation.getArguments()[0]; + ActionListener> listener = invocation.getArgument(0); listener.onResponse(List.of(defaultSparse, defaultChat, defaultText)); return Void.TYPE; }).when(service).defaultConfigs(any()); diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java index c071c60af716c..af1acc7530dce 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java @@ -43,10 +43,12 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.XContentFormattedException; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; @@ -79,6 +81,7 @@ public class ServerSentEventsRestActionListenerTests extends ESIntegTestCase { private static final String REQUEST_COUNT = "request_count"; private static final String WITH_ERROR = "with_error"; private static final String ERROR_ROUTE = "/_inference_error"; + private static final String FORMATTED_ERROR_ROUTE = "/_formatted_inference_error"; private static final String NO_STREAM_ROUTE = "/_inference_no_stream"; private static final Exception expectedException = new IllegalStateException("hello there"); private static final String expectedExceptionAsServerSentEvent = """ @@ -87,6 +90,11 @@ public class ServerSentEventsRestActionListenerTests extends ESIntegTestCase { "type":"illegal_state_exception","reason":"hello there"},"status":500\ }"""; + private static final Exception expectedFormattedException = new XContentFormattedException( + expectedException, + RestStatus.INTERNAL_SERVER_ERROR + ); + @Override protected boolean addMockHttpTransport() { return false; @@ -144,6 +152,16 @@ public List routes() { public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) { new ServerSentEventsRestActionListener(channel, threadPool).onFailure(expectedException); } + }, new RestHandler() { + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.POST, FORMATTED_ERROR_ROUTE)); + } + + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) { + new ServerSentEventsRestActionListener(channel, threadPool).onFailure(expectedFormattedException); + } }, new RestHandler() { @Override public List routes() { @@ -423,6 +441,21 @@ public void testErrorMidStream() { assertThat(collector.stringsVerified.getLast(), equalTo(expectedExceptionAsServerSentEvent)); } + public void testFormattedError() throws IOException { + var request = new Request(RestRequest.Method.POST.name(), FORMATTED_ERROR_ROUTE); + + try { + getRestClient().performRequest(request); + fail("Expected an exception to be thrown from the error route"); + } catch (ResponseException e) { + var response = e.getResponse(); + assertThat(response.getStatusLine().getStatusCode(), is(HttpStatus.SC_INTERNAL_SERVER_ERROR)); + assertThat(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8), equalTo(""" + \uFEFFevent: error + data:\s""" + expectedExceptionAsServerSentEvent + "\n\n")); + } + } + public void testNoStream() { var collector = new RandomStringCollector(); var expectedTestCount = randomIntBetween(2, 30); diff --git a/x-pack/plugin/inference/src/main/config/log4j2.properties b/x-pack/plugin/inference/src/main/config/log4j2.properties new file mode 100644 index 0000000000000..0b3b7bcc988c4 --- /dev/null +++ b/x-pack/plugin/inference/src/main/config/log4j2.properties @@ -0,0 +1,3 @@ +logger.entitlements_inference.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-inference.software.amazon.awssdk.profiles +logger.entitlements_inference.level = error + diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceException.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceException.java new file mode 100644 index 0000000000000..3caa5bd06d058 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceException.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.rest.RestStatus; + +public class InferenceException extends ElasticsearchException { + public InferenceException(String message, Throwable cause, Object... args) { + super(message, cause, args); + } + + @Override + public RestStatus status() { + // Override status so that we get the status of the cause while retaining the message of the inference exception when emitting to + // XContent + return ExceptionsHelper.status(getCause()); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 1afb6ca50a237..f840ebd9ed283 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -16,6 +16,7 @@ import java.util.Set; +import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_FILTER_FIX; import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED; import static org.elasticsearch.xpack.inference.queries.SemanticMatchQueryRewriteInterceptor.SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED; import static org.elasticsearch.xpack.inference.queries.SemanticSparseVectorQueryRewriteInterceptor.SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED; @@ -47,6 +48,7 @@ public Set getTestFeatures() { SemanticTextFieldMapper.SEMANTIC_TEXT_DELETE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ZERO_SIZE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX, + SemanticTextFieldMapper.SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS, SEMANTIC_TEXT_HIGHLIGHTER, SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED, SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, @@ -54,7 +56,8 @@ public Set getTestFeatures() { SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX, SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT, - SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT + SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT, + SEMANTIC_KNN_FILTER_FIX ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 6fc9870034018..2ff9fead27ec9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticRerankerServiceSettings; @@ -75,6 +76,8 @@ import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings; import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankServiceSettings; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankTaskSettings; import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; @@ -364,6 +367,17 @@ private static void addIbmWatsonxNamedWritables(List namedWriteables) { @@ -605,5 +619,12 @@ private static void addEisNamedWriteables(List nam ElasticInferenceServiceSparseEmbeddingsServiceSettings::new ) ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + ElasticInferenceServiceCompletionServiceSettings.NAME, + ElasticInferenceServiceCompletionServiceSettings::new + ) + ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index a3aaf8127d935..2df3f2d0a9891 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -34,6 +34,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.node.PluginComponentBinding; import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; @@ -58,6 +59,7 @@ import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; import org.elasticsearch.xpack.core.inference.action.GetInferenceServicesAction; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; @@ -67,6 +69,7 @@ import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportGetInferenceServicesAction; import org.elasticsearch.xpack.inference.action.TransportInferenceAction; +import org.elasticsearch.xpack.inference.action.TransportInferenceActionProxy; import org.elasticsearch.xpack.inference.action.TransportInferenceUsageAction; import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportUnifiedCompletionInferenceAction; @@ -104,7 +107,6 @@ import org.elasticsearch.xpack.inference.rest.RestInferenceAction; import org.elasticsearch.xpack.inference.rest.RestPutInferenceModelAction; import org.elasticsearch.xpack.inference.rest.RestStreamInferenceAction; -import org.elasticsearch.xpack.inference.rest.RestUnifiedCompletionInferenceAction; import org.elasticsearch.xpack.inference.rest.RestUpdateInferenceModelAction; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchService; @@ -116,7 +118,7 @@ import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceComponents; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettings; -import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationHandler; +import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationRequestHandler; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService; import org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiService; @@ -145,7 +147,8 @@ public class InferencePlugin extends Plugin SystemIndexPlugin, MapperPlugin, SearchPlugin, - InternalSearchPlugin { + InternalSearchPlugin, + ClusterPlugin { /** * When this setting is true the verification check that @@ -195,6 +198,7 @@ public InferencePlugin(Settings settings) { public List> getActions() { return List.of( new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class), + new ActionHandler<>(InferenceActionProxy.INSTANCE, TransportInferenceActionProxy.class), new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class), new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class), new ActionHandler<>(UpdateInferenceModelAction.INSTANCE, TransportUpdateInferenceModelAction.class), @@ -226,8 +230,7 @@ public List getRestHandlers( new RestUpdateInferenceModelAction(), new RestDeleteInferenceEndpointAction(), new RestGetInferenceDiagnosticsAction(), - new RestGetInferenceServicesAction(), - new RestUnifiedCompletionInferenceAction(threadPoolSetOnce) + new RestGetInferenceServicesAction() ); } @@ -270,14 +273,11 @@ public Collection createComponents(PluginServices services) { ); elasicInferenceServiceFactory.set(elasticInferenceServiceRequestSenderFactory); - ElasticInferenceServiceSettings inferenceServiceSettings = new ElasticInferenceServiceSettings(settings); - String elasticInferenceUrl = inferenceServiceSettings.getElasticInferenceServiceUrl(); + var inferenceServiceSettings = new ElasticInferenceServiceSettings(settings); + inferenceServiceSettings.init(services.clusterService()); - var elasticInferenceServiceComponentsInstance = new ElasticInferenceServiceComponents(elasticInferenceUrl); - elasticInferenceServiceComponents.set(elasticInferenceServiceComponentsInstance); - - var authorizationHandler = new ElasticInferenceServiceAuthorizationHandler( - elasticInferenceServiceComponentsInstance.elasticInferenceServiceUrl(), + var authorizationHandler = new ElasticInferenceServiceAuthorizationRequestHandler( + inferenceServiceSettings.getElasticInferenceServiceUrl(), services.threadPool() ); @@ -286,7 +286,7 @@ public Collection createComponents(PluginServices services) { context -> new ElasticInferenceService( elasicInferenceServiceFactory.get(), serviceComponents.get(), - elasticInferenceServiceComponentsInstance, + inferenceServiceSettings, modelRegistry, authorizationHandler ) @@ -309,7 +309,7 @@ public Collection createComponents(PluginServices services) { } inferenceServiceRegistry.set(serviceRegistry); - var actionFilter = new ShardBulkInferenceActionFilter(services.clusterService(), serviceRegistry, modelRegistry); + var actionFilter = new ShardBulkInferenceActionFilter(services.clusterService(), serviceRegistry, modelRegistry, getLicenseState()); shardBulkInferenceActionFilter.set(actionFilter); var meterRegistry = services.telemetryProvider().getMeterRegistry(); @@ -331,7 +331,6 @@ public Collection createComponents(PluginServices services) { // Add binding for interface -> implementation components.add(new PluginComponentBinding<>(InferenceServiceRateLimitCalculator.class, calculator)); - components.add(calculator); return components; } @@ -516,6 +515,15 @@ private String getElasticInferenceServiceUrl(ElasticInferenceServiceSettings set return settings.getElasticInferenceServiceUrl(); } + @Override + public void onNodeStarted() { + var registry = inferenceServiceRegistry.get(); + + if (registry != null) { + registry.onNodeStarted(); + } + } + protected SSLService getSslService() { return XPackPlugin.getSharedSslService(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java index 08d74a36d6503..b390a51f6d3e2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java @@ -50,12 +50,13 @@ import java.io.IOException; import java.util.Random; import java.util.concurrent.Executor; +import java.util.concurrent.Flow; import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.ExceptionsHelper.unwrapCause; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE; -import static org.elasticsearch.xpack.inference.common.InferenceAPIClusterAwareRateLimitingFeature.INFERENCE_API_CLUSTER_AWARE_RATE_LIMITING_FEATURE_FLAG; import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.modelAttributes; import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.responseAttributes; @@ -188,10 +189,6 @@ private void validateRequest(Request request, UnparsedModel unparsedModel) { } private NodeRoutingDecision determineRouting(String serviceName, Request request, UnparsedModel unparsedModel) { - if (INFERENCE_API_CLUSTER_AWARE_RATE_LIMITING_FEATURE_FLAG.isEnabled() == false) { - return NodeRoutingDecision.handleLocally(); - } - var modelTaskType = unparsedModel.taskType(); // Rerouting not supported or request was already rerouted @@ -285,7 +282,9 @@ private void inferOnServiceWithMetrics( var instrumentedStream = new PublisherWithMetrics(timer, model); taskProcessor.subscribe(instrumentedStream); - listener.onResponse(new InferenceAction.Response(inferenceResults, instrumentedStream)); + var streamErrorHandler = streamErrorHandler(instrumentedStream); + + listener.onResponse(new InferenceAction.Response(inferenceResults, streamErrorHandler)); } else { recordMetrics(model, timer, null); listener.onResponse(new InferenceAction.Response(inferenceResults)); @@ -296,9 +295,13 @@ private void inferOnServiceWithMetrics( })); } + protected Flow.Publisher streamErrorHandler(Flow.Processor upstream) { + return upstream; + } + private void recordMetrics(Model model, InferenceTimer timer, @Nullable Throwable t) { try { - inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(model, t)); + inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(model, unwrapCause(t))); } catch (Exception e) { log.atDebug().withThrowable(e).log("Failed to record metrics with a parsed model, dropping metrics"); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java index 0e441e78fb986..be492b3c0b52c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java @@ -18,8 +18,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.UnparsedModel; @@ -53,7 +53,6 @@ public TransportDeleteInferenceEndpointAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry ) { @@ -109,6 +108,18 @@ private void doExecuteForked( if (errorString != null) { listener.onFailure(new ElasticsearchStatusException(errorString, RestStatus.CONFLICT)); return; + } else if (isInferenceIdReserved(request.getInferenceEndpointId())) { + listener.onFailure( + new ElasticsearchStatusException( + Strings.format( + "[%s] is a reserved inference endpoint. Use the force=true query parameter " + + "to delete the inference endpoint.", + request.getInferenceEndpointId() + ), + RestStatus.BAD_REQUEST + ) + ); + return; } } @@ -177,6 +188,10 @@ private static String endpointIsReferencedInPipelinesOrIndexes(final ClusterStat return null; } + private boolean isInferenceIdReserved(String inferenceEndpointId) { + return modelRegistry.containsDefaultConfigId(inferenceEndpointId); + } + private static String buildErrorString(String inferenceEndpointId, Set pipelines, Set indexes) { StringBuilder errorString = new StringBuilder(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionProxy.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionProxy.java new file mode 100644 index 0000000000000..6d46f834d4873 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionProxy.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; + +import java.io.IOException; + +import static org.elasticsearch.xpack.core.ClientHelper.INFERENCE_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public class TransportInferenceActionProxy extends HandledTransportAction { + private final ModelRegistry modelRegistry; + private final Client client; + + @Inject + public TransportInferenceActionProxy( + TransportService transportService, + ActionFilters actionFilters, + ModelRegistry modelRegistry, + Client client + ) { + super( + InferenceActionProxy.NAME, + transportService, + actionFilters, + InferenceActionProxy.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + + this.modelRegistry = modelRegistry; + this.client = client; + } + + @Override + protected void doExecute(Task task, InferenceActionProxy.Request request, ActionListener listener) { + try { + ActionListener getModelListener = listener.delegateFailureAndWrap((l, unparsedModel) -> { + if (unparsedModel.taskType() == TaskType.CHAT_COMPLETION) { + sendUnifiedCompletionRequest(request, l); + } else { + sendInferenceActionRequest(request, l); + } + }); + + if (request.getTaskType() == TaskType.ANY) { + modelRegistry.getModelWithSecrets(request.getInferenceEntityId(), getModelListener); + } else if (request.getTaskType() == TaskType.CHAT_COMPLETION) { + sendUnifiedCompletionRequest(request, listener); + } else { + sendInferenceActionRequest(request, listener); + } + } catch (Exception e) { + listener.onFailure(e); + } + } + + private void sendUnifiedCompletionRequest(InferenceActionProxy.Request request, ActionListener listener) { + // format any validation exceptions from the rest -> transport path as UnifiedChatCompletionException + var unifiedErrorFormatListener = listener.delegateResponse((l, e) -> l.onFailure(UnifiedChatCompletionException.fromThrowable(e))); + + try { + if (request.isStreaming() == false) { + throw new ElasticsearchStatusException( + "The [chat_completion] task type only supports streaming, please try again with the _stream API", + RestStatus.BAD_REQUEST + ); + } + + UnifiedCompletionAction.Request unifiedRequest; + try ( + var parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, request.getContent(), request.getContentType()) + ) { + unifiedRequest = UnifiedCompletionAction.Request.parseRequest( + request.getInferenceEntityId(), + request.getTaskType(), + request.getTimeout(), + parser + ); + } + + executeAsyncWithOrigin(client, INFERENCE_ORIGIN, UnifiedCompletionAction.INSTANCE, unifiedRequest, unifiedErrorFormatListener); + } catch (Exception e) { + unifiedErrorFormatListener.onFailure(e); + } + } + + private void sendInferenceActionRequest(InferenceActionProxy.Request request, ActionListener listener) + throws IOException { + InferenceAction.Request.Builder inferenceActionRequestBuilder; + try (var parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, request.getContent(), request.getContentType())) { + inferenceActionRequestBuilder = InferenceAction.Request.parseRequest( + request.getInferenceEntityId(), + request.getTaskType(), + parser + ); + inferenceActionRequestBuilder.setInferenceTimeout(request.getTimeout()).setStream(request.isStreaming()); + } + + executeAsyncWithOrigin(client, INFERENCE_ORIGIN, InferenceAction.INSTANCE, inferenceActionRequestBuilder.build(), listener); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java index 2e3090f2afd59..1144a11d86cc9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.InferenceServiceResults; @@ -20,14 +21,19 @@ import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; import org.elasticsearch.xpack.inference.common.InferenceServiceRateLimitCalculator; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; +import java.util.concurrent.Flow; + public class TransportUnifiedCompletionInferenceAction extends BaseTransportInferenceAction { @Inject @@ -86,4 +92,40 @@ protected void doInference( ) { service.unifiedCompletionInfer(model, request.getUnifiedCompletionRequest(), null, listener); } + + @Override + protected void doExecute(Task task, UnifiedCompletionAction.Request request, ActionListener listener) { + super.doExecute(task, request, listener.delegateResponse((l, e) -> l.onFailure(UnifiedChatCompletionException.fromThrowable(e)))); + } + + /** + * If we get any errors, either in {@link #doExecute} via the listener.onFailure or while streaming, make sure that they are formatted + * as {@link UnifiedChatCompletionException}. + */ + @Override + protected Flow.Publisher streamErrorHandler(Flow.Processor upstream) { + return downstream -> { + upstream.subscribe(new Flow.Subscriber<>() { + @Override + public void onSubscribe(Flow.Subscription subscription) { + downstream.onSubscribe(subscription); + } + + @Override + public void onNext(ChunkedToXContent item) { + downstream.onNext(item); + } + + @Override + public void onError(Throwable throwable) { + downstream.onError(UnifiedChatCompletionException.fromThrowable(throwable)); + } + + @Override + public void onComplete() { + downstream.onComplete(); + } + }); + }; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java index bb8582178dbb8..c6894c7f45ced 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; @@ -51,6 +52,7 @@ import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalModel; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; @@ -257,14 +259,13 @@ private void updateInClusterEndpoint( ActionListener listener ) throws IOException { // The model we are trying to update must have a trained model associated with it if it is an in-cluster deployment - throwIfTrainedModelDoesntExist(request); + var deploymentId = getDeploymentIdForInClusterEndpoint(existingParsedModel); + throwIfTrainedModelDoesntExist(request.getInferenceEntityId(), deploymentId); Map serviceSettings = request.getContentAsSettings().serviceSettings(); if (serviceSettings != null && serviceSettings.get(NUM_ALLOCATIONS) instanceof Integer numAllocations) { - UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request( - request.getInferenceEntityId() - ); + UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId); updateRequest.setNumberOfAllocations(numAllocations); var delegate = listener.delegateFailure((l2, response) -> { @@ -272,7 +273,8 @@ private void updateInClusterEndpoint( }); logger.info( - "Updating trained model deployment for inference entity [{}] with [{}] num_allocations", + "Updating trained model deployment [{}] for inference entity [{}] with [{}] num_allocations", + deploymentId, request.getInferenceEntityId(), numAllocations ); @@ -295,12 +297,26 @@ private boolean isInClusterService(String name) { return List.of(ElasticsearchInternalService.NAME, ElasticsearchInternalService.OLD_ELSER_SERVICE_NAME).contains(name); } - private void throwIfTrainedModelDoesntExist(UpdateInferenceModelAction.Request request) throws ElasticsearchStatusException { - var assignments = TrainedModelAssignmentUtils.modelAssignments(request.getInferenceEntityId(), clusterService.state()); + private String getDeploymentIdForInClusterEndpoint(Model model) { + if (model instanceof ElasticsearchInternalModel esModel) { + return esModel.mlNodeDeploymentId(); + } else { + throw new IllegalStateException( + Strings.format( + "Cannot update inference endpoint [%s]. Class [%s] is not an Elasticsearch internal model", + model.getInferenceEntityId(), + model.getClass().getSimpleName() + ) + ); + } + } + + private void throwIfTrainedModelDoesntExist(String inferenceEntityId, String deploymentId) throws ElasticsearchStatusException { + var assignments = TrainedModelAssignmentUtils.modelAssignments(deploymentId, clusterService.state()); if ((assignments == null || assignments.isEmpty())) { throw ExceptionsHelper.entityNotFoundException( Messages.MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE, - request.getInferenceEntityId() + inferenceEntityId ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java index 3933260664b7c..dca803696ddf2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.action.filter; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; @@ -38,10 +37,14 @@ import org.elasticsearch.inference.MinimalServiceSettings; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.xcontent.XContent; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; +import org.elasticsearch.xpack.inference.InferenceException; import org.elasticsearch.xpack.inference.mapper.SemanticTextField; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; import org.elasticsearch.xpack.inference.mapper.SemanticTextUtils; @@ -58,6 +61,8 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE; + /** * A {@link MappedActionFilter} that intercepts {@link BulkShardRequest} to apply inference on fields specified * as {@link SemanticTextFieldMapper} in the index mapping. For each semantic text field referencing fields in @@ -76,25 +81,29 @@ public class ShardBulkInferenceActionFilter implements MappedActionFilter { private final ClusterService clusterService; private final InferenceServiceRegistry inferenceServiceRegistry; private final ModelRegistry modelRegistry; + private final XPackLicenseState licenseState; private final int batchSize; public ShardBulkInferenceActionFilter( ClusterService clusterService, InferenceServiceRegistry inferenceServiceRegistry, - ModelRegistry modelRegistry + ModelRegistry modelRegistry, + XPackLicenseState licenseState ) { - this(clusterService, inferenceServiceRegistry, modelRegistry, DEFAULT_BATCH_SIZE); + this(clusterService, inferenceServiceRegistry, modelRegistry, licenseState, DEFAULT_BATCH_SIZE); } public ShardBulkInferenceActionFilter( ClusterService clusterService, InferenceServiceRegistry inferenceServiceRegistry, ModelRegistry modelRegistry, + XPackLicenseState licenseState, int batchSize ) { this.clusterService = clusterService; this.inferenceServiceRegistry = inferenceServiceRegistry; this.modelRegistry = modelRegistry; + this.licenseState = licenseState; this.batchSize = batchSize; } @@ -279,7 +288,7 @@ public void onFailure(Exception exc) { request.field ); } else { - failure = new ElasticsearchException( + failure = new InferenceException( "Error loading inference for inference id [{}] on field [{}]", exc, inferenceId, @@ -308,7 +317,7 @@ public void onResponse(List results) { var acc = inferenceResults.get(request.index); if (result instanceof ChunkedInferenceError error) { acc.addFailure( - new ElasticsearchException( + new InferenceException( "Exception when running inference id [{}] on field [{}]", error.exception(), inferenceProvider.model.getInferenceEntityId(), @@ -340,7 +349,7 @@ public void onFailure(Exception exc) { for (FieldInferenceRequest request : requests) { addInferenceResponseFailure( request.index, - new ElasticsearchException( + new InferenceException( "Exception when running inference id [{}] on field [{}]", exc, inferenceProvider.model.getInferenceEntityId(), @@ -482,7 +491,7 @@ private Map> createFieldInferenceRequests(Bu isUpdateRequest = true; if (updateRequest.script() != null) { addInferenceResponseFailure( - item.id(), + itemIndex, new ElasticsearchStatusException( "Cannot apply update with a script on indices that contain [{}] field(s)", RestStatus.BAD_REQUEST, @@ -540,7 +549,7 @@ private Map> createFieldInferenceRequests(Bu if (valueObj == null || valueObj == EXPLICIT_NULL) { if (isUpdateRequest && useLegacyFormat) { addInferenceResponseFailure( - item.id(), + itemIndex, new ElasticsearchStatusException( "Field [{}] must be specified on an update request to calculate inference for field [{}]", RestStatus.BAD_REQUEST, @@ -557,7 +566,12 @@ private Map> createFieldInferenceRequests(Bu try { values = SemanticTextUtils.nodeStringValues(field, valueObj); } catch (Exception exc) { - addInferenceResponseFailure(item.id(), exc); + addInferenceResponseFailure(itemIndex, exc); + break; + } + + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + addInferenceResponseFailure(itemIndex, LicenseUtils.newComplianceException(XPackField.INFERENCE)); break; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java index 9d6f5bb89218f..0033cc9ee2bef 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java @@ -134,11 +134,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; SentenceBoundaryChunkingSettings that = (SentenceBoundaryChunkingSettings) o; - return Objects.equals(maxChunkSize, that.maxChunkSize); + return Objects.equals(maxChunkSize, that.maxChunkSize) && Objects.equals(sentenceOverlap, that.sentenceOverlap); } @Override public int hashCode() { - return Objects.hash(maxChunkSize); + return Objects.hash(maxChunkSize, sentenceOverlap); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java index 27d1f1bd14e2c..61cada6e75ef1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java @@ -12,12 +12,9 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; -import java.net.URI; - public class ActionUtils { public static ActionListener wrapFailuresInElasticsearchException( @@ -30,7 +27,12 @@ public static ActionListener wrapFailuresInElasticsearc if (unwrappedException instanceof ElasticsearchException esException) { l.onFailure(esException); } else { - l.onFailure(createInternalServerError(unwrappedException, errorMessage)); + l.onFailure( + createInternalServerError( + unwrappedException, + Strings.format("%s. Cause: %s", errorMessage, unwrappedException.getMessage()) + ) + ); } }); } @@ -39,11 +41,7 @@ public static ElasticsearchStatusException createInternalServerError(Throwable e return new ElasticsearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR, e); } - public static String constructFailedToSendRequestMessage(@Nullable URI uri, String message) { - if (uri != null) { - return Strings.format("Failed to send %s request to [%s]", message, uri); - } - + public static String constructFailedToSendRequestMessage(String message) { return Strings.format("Failed to send %s request", message); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionAction.java index dc1d31f3e59df..509d360291deb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionAction.java @@ -14,12 +14,11 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount; import org.elasticsearch.xpack.inference.external.http.sender.AlibabaCloudSearchCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -27,7 +26,6 @@ import java.util.Objects; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; @@ -45,24 +43,14 @@ public AlibabaCloudSearchCompletionAction(Sender sender, AlibabaCloudSearchCompl this.model = Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey()); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search completion"); + this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("AlibabaCloud Search completion"); this.requestCreator = AlibabaCloudSearchCompletionRequestManager.of(account, model, serviceComponents.threadPool()); } @Override public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure( - new ElasticsearchStatusException( - format("Invalid inference input type, task type [%s] do not support Field [query]", TaskType.COMPLETION), - RestStatus.INTERNAL_SERVER_ERROR - ) - ); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() % 2 == 0) { + var completionInput = inferenceInputs.castTo(ChatCompletionInput.class); + if (completionInput.getInputs().size() % 2 == 0) { listener.onFailure( new ElasticsearchStatusException( "Alibaba Completion's inputs must be an odd number. The last input is the current query, " diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java index 7a22bbf6b4bfd..91bb582165d3f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java @@ -36,7 +36,7 @@ public AlibabaCloudSearchEmbeddingsAction(Sender sender, AlibabaCloudSearchEmbed this.model = Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey()); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search text embeddings"); + this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("AlibabaCloud Search text embeddings"); this.requestCreator = AlibabaCloudSearchEmbeddingsRequestManager.of(account, model, serviceComponents.threadPool()); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java index 88229ce63463b..c73003d454834 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java @@ -39,7 +39,7 @@ public class AlibabaCloudSearchRerankAction implements ExecutableAction { public AlibabaCloudSearchRerankAction(Sender sender, AlibabaCloudSearchRerankModel model, ServiceComponents serviceComponents) { this.model = Objects.requireNonNull(model); this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey()); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search rerank"); + this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("AlibabaCloud Search rerank"); this.sender = Objects.requireNonNull(sender); this.requestCreator = AlibabaCloudSearchRerankRequestManager.of(account, model, serviceComponents.threadPool()); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java index 2cd31ff83d200..f76d21214c628 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java @@ -39,7 +39,7 @@ public class AlibabaCloudSearchSparseAction implements ExecutableAction { public AlibabaCloudSearchSparseAction(Sender sender, AlibabaCloudSearchSparseModel model, ServiceComponents serviceComponents) { this.model = Objects.requireNonNull(model); this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey()); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search sparse embeddings"); + this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("AlibabaCloud Search sparse embeddings"); this.sender = Objects.requireNonNull(sender); requestCreator = AlibabaCloudSearchSparseRequestManager.of(account, model, serviceComponents.threadPool()); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java index 2715298c22d63..c66eacfcdd8e7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java @@ -43,7 +43,7 @@ public ExecutableAction create(AmazonBedrockEmbeddingsModel embeddingsModel, Map serviceComponents.threadPool(), timeout ); - var errorMessage = constructFailedToSendRequestMessage(null, "Amazon Bedrock embeddings"); + var errorMessage = constructFailedToSendRequestMessage("Amazon Bedrock embeddings"); return new SenderExecutableAction(sender, requestManager, errorMessage); } @@ -51,7 +51,7 @@ public ExecutableAction create(AmazonBedrockEmbeddingsModel embeddingsModel, Map public ExecutableAction create(AmazonBedrockChatCompletionModel completionModel, Map taskSettings) { var overriddenModel = AmazonBedrockChatCompletionModel.of(completionModel, taskSettings); var requestManager = new AmazonBedrockChatCompletionRequestManager(overriddenModel, serviceComponents.threadPool(), timeout); - var errorMessage = constructFailedToSendRequestMessage(null, "Amazon Bedrock completion"); + var errorMessage = constructFailedToSendRequestMessage("Amazon Bedrock completion"); return new SenderExecutableAction(sender, requestManager, errorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java index aea6d065e09d1..a97367e29853e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java @@ -36,7 +36,7 @@ public AnthropicActionCreator(Sender sender, ServiceComponents serviceComponents public ExecutableAction create(AnthropicChatCompletionModel model, Map taskSettings) { var overriddenModel = AnthropicChatCompletionModel.of(model, taskSettings); var requestCreator = AnthropicCompletionRequestManager.of(overriddenModel, serviceComponents.threadPool()); - var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getUri(), ERROR_PREFIX); + var errorMessage = constructFailedToSendRequestMessage(ERROR_PREFIX); return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java index 6a80cee3afd57..146fe5e6b128d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java @@ -34,7 +34,7 @@ public AzureAiStudioActionCreator(Sender sender, ServiceComponents serviceCompon public ExecutableAction create(AzureAiStudioChatCompletionModel completionModel, Map taskSettings) { var overriddenModel = AzureAiStudioChatCompletionModel.of(completionModel, taskSettings); var requestManager = new AzureAiStudioChatCompletionRequestManager(overriddenModel, serviceComponents.threadPool()); - var errorMessage = constructFailedToSendRequestMessage(completionModel.uri(), "Azure AI Studio completion"); + var errorMessage = constructFailedToSendRequestMessage("Azure AI Studio completion"); return new SenderExecutableAction(sender, requestManager, errorMessage); } @@ -46,7 +46,7 @@ public ExecutableAction create(AzureAiStudioEmbeddingsModel embeddingsModel, Map serviceComponents.truncator(), serviceComponents.threadPool() ); - var errorMessage = constructFailedToSendRequestMessage(embeddingsModel.uri(), "Azure AI Studio embeddings"); + var errorMessage = constructFailedToSendRequestMessage("Azure AI Studio embeddings"); return new SenderExecutableAction(sender, requestManager, errorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java index 1454b7c92ad91..88803bd15d4e3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java @@ -43,7 +43,7 @@ public ExecutableAction create(AzureOpenAiEmbeddingsModel model, Map taskSettings) { var overriddenModel = AzureOpenAiCompletionModel.of(model, taskSettings); var requestCreator = new AzureOpenAiCompletionRequestManager(overriddenModel, serviceComponents.threadPool()); - var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getUri(), COMPLETION_ERROR_PREFIX); + var errorMessage = constructFailedToSendRequestMessage(COMPLETION_ERROR_PREFIX); return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, COMPLETION_ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java index 9462ab1a361b4..b39a63dae2710 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java @@ -42,10 +42,7 @@ public CohereActionCreator(Sender sender, ServiceComponents serviceComponents) { @Override public ExecutableAction create(CohereEmbeddingsModel model, Map taskSettings, InputType inputType) { var overriddenModel = CohereEmbeddingsModel.of(model, taskSettings, inputType); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - overriddenModel.getServiceSettings().getCommonSettings().uri(), - "Cohere embeddings" - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Cohere embeddings"); // TODO - Batching pass the batching class on to the CohereEmbeddingsRequestManager var requestCreator = CohereEmbeddingsRequestManager.of(overriddenModel, serviceComponents.threadPool()); return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); @@ -55,10 +52,7 @@ public ExecutableAction create(CohereEmbeddingsModel model, Map public ExecutableAction create(CohereRerankModel model, Map taskSettings) { var overriddenModel = CohereRerankModel.of(model, taskSettings); var requestCreator = CohereRerankRequestManager.of(overriddenModel, serviceComponents.threadPool()); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - overriddenModel.getServiceSettings().uri(), - "Cohere rerank" - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Cohere rerank"); return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } @@ -66,10 +60,7 @@ public ExecutableAction create(CohereRerankModel model, Map task public ExecutableAction create(CohereCompletionModel model, Map taskSettings) { // no overridden model as task settings are always empty for cohere completion model var requestManager = CohereCompletionRequestManager.of(model, serviceComponents.threadPool()); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - model.getServiceSettings().uri(), - COMPLETION_ERROR_PREFIX - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(COMPLETION_ERROR_PREFIX); return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, COMPLETION_ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java index 915d8b3b64bc1..8a5c735b96974 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java @@ -48,7 +48,6 @@ public ElasticInferenceServiceActionCreator( public ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model) { var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents, traceContext, inputType); var errorMessage = constructFailedToSendRequestMessage( - model.uri(), String.format(Locale.ROOT, "%s sparse embeddings", ELASTIC_INFERENCE_SERVICE_IDENTIFIER) ); return new SenderExecutableAction(sender, requestManager, errorMessage); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java index 99f535f81485c..125e16cd2a58a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java @@ -41,13 +41,13 @@ public ExecutableAction create(GoogleVertexAiEmbeddingsModel model, Map taskSettings) { - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI rerank"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google Vertex AI rerank"); var requestManager = GoogleVertexAiRerankRequestManager.of(model, serviceComponents.threadPool()); return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionCreator.java index 7cad7c42bdcf1..9eb4ab2024aed 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionCreator.java @@ -12,9 +12,11 @@ import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.IbmWatsonxEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.IbmWatsonxRerankRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; import java.util.Map; import java.util.Objects; @@ -22,7 +24,6 @@ import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; public class IbmWatsonxActionCreator implements IbmWatsonxActionVisitor { - private final Sender sender; private final ServiceComponents serviceComponents; @@ -33,7 +34,7 @@ public IbmWatsonxActionCreator(Sender sender, ServiceComponents serviceComponent @Override public ExecutableAction create(IbmWatsonxEmbeddingsModel model, Map taskSettings) { - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "IBM WatsonX embeddings"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("IBM WatsonX embeddings"); return new SenderExecutableAction( sender, getEmbeddingsRequestManager(model, serviceComponents.truncator(), serviceComponents.threadPool()), @@ -41,6 +42,14 @@ public ExecutableAction create(IbmWatsonxEmbeddingsModel model, Map taskSettings) { + var overriddenModel = IbmWatsonxRerankModel.of(model, taskSettings); + var requestCreator = IbmWatsonxRerankRequestManager.of(overriddenModel, serviceComponents.threadPool()); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Ibm Watsonx rerank"); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); + } + protected IbmWatsonxEmbeddingsRequestManager getEmbeddingsRequestManager( IbmWatsonxEmbeddingsModel model, Truncator truncator, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionVisitor.java index 0a13ec2fb4645..474533040e0c3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionVisitor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxActionVisitor.java @@ -9,9 +9,12 @@ import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; import java.util.Map; public interface IbmWatsonxActionVisitor { ExecutableAction create(IbmWatsonxEmbeddingsModel model, Map taskSettings); + + ExecutableAction create(IbmWatsonxRerankModel model, Map taskSettings); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java index 4d5827a3bf266..590079fd4dc50 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java @@ -37,10 +37,7 @@ public JinaAIActionCreator(Sender sender, ServiceComponents serviceComponents) { @Override public ExecutableAction create(JinaAIEmbeddingsModel model, Map taskSettings, InputType inputType) { var overriddenModel = JinaAIEmbeddingsModel.of(model, taskSettings, inputType); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - overriddenModel.getServiceSettings().getCommonSettings().uri(), - "JinaAI embeddings" - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("JinaAI embeddings"); var requestCreator = JinaAIEmbeddingsRequestManager.of(overriddenModel, serviceComponents.threadPool()); return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } @@ -48,10 +45,7 @@ public ExecutableAction create(JinaAIEmbeddingsModel model, Map @Override public ExecutableAction create(JinaAIRerankModel model, Map taskSettings) { var overriddenModel = JinaAIRerankModel.of(model, taskSettings); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - overriddenModel.getServiceSettings().getCommonSettings().uri(), - "JinaAI rerank" - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("JinaAI rerank"); var requestCreator = JinaAIRerankRequestManager.of(overriddenModel, serviceComponents.threadPool()); return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java index 21a80ee9d21fa..a1c4764a95091 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java @@ -35,7 +35,7 @@ public ExecutableAction create(MistralEmbeddingsModel embeddingsModel, Map serviceComponents.truncator(), serviceComponents.threadPool() ); - var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getServiceSettings().uri(), "OpenAI embeddings"); + var errorMessage = constructFailedToSendRequestMessage("OpenAI embeddings"); return new SenderExecutableAction(sender, requestCreator, errorMessage); } @@ -51,7 +51,7 @@ public ExecutableAction create(OpenAiEmbeddingsModel model, Map public ExecutableAction create(OpenAiChatCompletionModel model, Map taskSettings) { var overriddenModel = OpenAiChatCompletionModel.of(model, taskSettings); var requestCreator = OpenAiCompletionRequestManager.of(overriddenModel, serviceComponents.threadPool()); - var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getServiceSettings().uri(), COMPLETION_ERROR_PREFIX); + var errorMessage = constructFailedToSendRequestMessage(COMPLETION_ERROR_PREFIX); return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, COMPLETION_ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java index c0bccb9b2cd49..9e9531af06c8f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java @@ -8,16 +8,23 @@ package org.elasticsearch.xpack.inference.external.elastic; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.openai.OpenAiUnifiedStreamingProcessor; import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceErrorResponseEntity; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; +import java.util.Locale; import java.util.concurrent.Flow; +import static org.elasticsearch.core.Strings.format; + public class ElasticInferenceServiceUnifiedChatCompletionResponseHandler extends ElasticInferenceServiceResponseHandler { public ElasticInferenceServiceUnifiedChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { super(requestType, parseFunction); @@ -31,10 +38,54 @@ public boolean canHandleStreamingResponses() { @Override public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); - var openAiProcessor = new OpenAiUnifiedStreamingProcessor(); // EIS uses the unified API spec + // EIS uses the unified API spec + var openAiProcessor = new OpenAiUnifiedStreamingProcessor((m, e) -> buildMidStreamError(request, m, e)); flow.subscribe(serverSentEventProcessor); serverSentEventProcessor.subscribe(openAiProcessor); return new StreamingUnifiedChatCompletionResults(openAiProcessor); } + + @Override + protected Exception buildError(String message, Request request, HttpResult result, ErrorResponse errorResponse) { + assert request.isStreaming() : "Only streaming requests support this format"; + var responseStatusCode = result.response().getStatusLine().getStatusCode(); + if (request.isStreaming()) { + var restStatus = toRestStatus(responseStatusCode); + return new UnifiedChatCompletionException( + restStatus, + errorMessage(message, request, result, errorResponse, responseStatusCode), + "error", + restStatus.name().toLowerCase(Locale.ROOT) + ); + } else { + return super.buildError(message, request, result, errorResponse); + } + } + + private static Exception buildMidStreamError(Request request, String message, Exception e) { + var errorResponse = ElasticInferenceServiceErrorResponseEntity.fromString(message); + if (errorResponse.errorStructureFound()) { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format( + "%s for request from inference entity id [%s]. Error message: [%s]", + SERVER_ERROR_OBJECT, + request.getInferenceEntityId(), + errorResponse.getErrorMessage() + ), + "error", + "stream_error" + ); + } else if (e != null) { + return UnifiedChatCompletionException.fromThrowable(e); + } else { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format("%s for request from inference entity id [%s]", SERVER_ERROR_OBJECT, request.getInferenceEntityId()), + "error", + "stream_error" + ); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java index f0102d01b37a1..7936e6779c8d5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.external.http; import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.concurrent.FutureCallback; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; @@ -26,7 +27,6 @@ import java.io.IOException; import java.util.Objects; import java.util.concurrent.CancellationException; -import java.util.concurrent.Flow; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.core.Strings.format; @@ -56,14 +56,18 @@ public static HttpClient create( PoolingNHttpClientConnectionManager connectionManager, ThrottlerManager throttlerManager ) { - CloseableHttpAsyncClient client = createAsyncClient(Objects.requireNonNull(connectionManager)); + var client = createAsyncClient(Objects.requireNonNull(connectionManager), Objects.requireNonNull(settings)); return new HttpClient(settings, client, threadPool, throttlerManager); } - private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConnectionManager connectionManager) { - HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create(); - clientBuilder.setConnectionManager(connectionManager); + private static CloseableHttpAsyncClient createAsyncClient( + PoolingNHttpClientConnectionManager connectionManager, + HttpSettings settings + ) { + var requestConfig = RequestConfig.custom().setConnectTimeout(settings.connectionTimeout()).build(); + + var clientBuilder = HttpAsyncClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig); // The apache client will be shared across all connections because it can be expensive to create it // so we don't want to support cookies to avoid accidental authentication for unauthorized users clientBuilder.disableCookieManagement(); @@ -149,7 +153,7 @@ private void failUsingUtilityThread(Exception exception, ActionListener liste threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> listener.onFailure(exception)); } - public void stream(HttpRequest request, HttpContext context, ActionListener> listener) throws IOException { + public void stream(HttpRequest request, HttpContext context, ActionListener listener) throws IOException { // The caller must call start() first before attempting to send a request assert status.get() == Status.STARTED : "call start() before attempting to send a request"; @@ -157,7 +161,7 @@ public void stream(HttpRequest request, HttpContext context, ActionListener client.execute(request.requestProducer(), streamingProcessor, context, new FutureCallback<>() { @Override - public void completed(HttpResponse response) { + public void completed(Void response) { streamingProcessor.close(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java index b2825d1b79cbf..c8d5b36a319ec 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.TimeValue; import java.util.List; import java.util.Objects; @@ -27,12 +28,21 @@ public class HttpSettings { Setting.Property.Dynamic ); + // The time we wait for a connection to establish + public static final Setting CONNECTION_TIMEOUT = Setting.timeSetting( + "xpack.inference.http.connect_timeout", + TimeValue.timeValueSeconds(5), + Setting.Property.NodeScope + ); + private volatile ByteSizeValue maxResponseSize; + private final int connectionTimeout; public HttpSettings(Settings settings, ClusterService clusterService) { Objects.requireNonNull(clusterService); Objects.requireNonNull(settings); maxResponseSize = MAX_HTTP_RESPONSE_SIZE.get(settings); + connectionTimeout = Math.toIntExact(CONNECTION_TIMEOUT.get(settings).getMillis()); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_HTTP_RESPONSE_SIZE, this::setMaxResponseSize); } @@ -41,11 +51,15 @@ public ByteSizeValue getMaxResponseSize() { return maxResponseSize; } + public int connectionTimeout() { + return connectionTimeout; + } + private void setMaxResponseSize(ByteSizeValue maxResponseSize) { this.maxResponseSize = maxResponseSize; } public static List> getSettingsDefinitions() { - return List.of(MAX_HTTP_RESPONSE_SIZE); + return List.of(MAX_HTTP_RESPONSE_SIZE, CONNECTION_TIMEOUT); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResult.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResult.java new file mode 100644 index 0000000000000..fb4b862b3becb --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResult.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http; + +import org.apache.http.HttpResponse; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; + +import java.io.ByteArrayOutputStream; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicReference; + +public record StreamingHttpResult(HttpResponse response, Flow.Publisher body) { + public boolean isSuccessfulResponse() { + var code = response.getStatusLine().getStatusCode(); + return code >= 200 && code < 300; + } + + public Flow.Publisher toHttpResult() { + return subscriber -> body().subscribe(new Flow.Subscriber<>() { + @Override + public void onSubscribe(Flow.Subscription subscription) { + subscriber.onSubscribe(subscription); + } + + @Override + public void onNext(byte[] item) { + subscriber.onNext(new HttpResult(response(), item)); + } + + @Override + public void onError(Throwable throwable) { + subscriber.onError(throwable); + } + + @Override + public void onComplete() { + subscriber.onComplete(); + } + }); + } + + public void readFullResponse(ActionListener fullResponse) { + var stream = new ByteArrayOutputStream(); + AtomicReference upstream = new AtomicReference<>(null); + body.subscribe(new Flow.Subscriber<>() { + @Override + public void onSubscribe(Flow.Subscription subscription) { + upstream.set(subscription); + upstream.get().request(1); + } + + @Override + public void onNext(byte[] item) { + stream.writeBytes(item); + upstream.get().request(1); + } + + @Override + public void onError(Throwable throwable) { + ExceptionsHelper.maybeError(throwable).ifPresent(ExceptionsHelper::maybeDieOnAnotherThread); + fullResponse.onFailure(new RuntimeException("Fatal while fully consuming stream", throwable)); + } + + @Override + public void onComplete() { + fullResponse.onResponse(new HttpResult(response, stream.toByteArray())); + } + }); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java index 0b2268a448c8a..62ac1ac8a56bc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java @@ -13,6 +13,7 @@ import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import org.apache.http.nio.util.SimpleInputBuffer; import org.apache.http.protocol.HttpContext; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.threadpool.ThreadPool; @@ -39,51 +40,31 @@ * so this publisher will send a single HttpResult. If the HttpResponse is healthy, Apache will send an HttpResponse with or without * the HttpEntity.

*/ -class StreamingHttpResultPublisher implements HttpAsyncResponseConsumer, Flow.Publisher { - private final HttpSettings settings; - private final ActionListener> listener; +class StreamingHttpResultPublisher implements HttpAsyncResponseConsumer { + private final ActionListener listener; private final AtomicBoolean listenerCalled = new AtomicBoolean(false); - // used to manage the HTTP response - private volatile HttpResponse response; - private volatile Exception ex; - - // used to control the state of this publisher (Apache) and its interaction with its subscriber private final AtomicBoolean isDone = new AtomicBoolean(false); private final AtomicBoolean subscriptionCanceled = new AtomicBoolean(false); - private volatile Flow.Subscriber subscriber; - - private final RequestBasedTaskRunner taskRunner; - private final AtomicBoolean pendingRequest = new AtomicBoolean(false); - private final Deque queue = new ConcurrentLinkedDeque<>(); - // used to control the flow of data from the Apache client, if we're producing more bytes than we can consume then we'll pause private final SimpleInputBuffer inputBuffer = new SimpleInputBuffer(4096); - private final AtomicLong bytesInQueue = new AtomicLong(0); - private final Object ioLock = new Object(); - private volatile IOControl savedIoControl; + private final DataPublisher publisher; + private final ApacheClientBackpressure backpressure; - StreamingHttpResultPublisher(ThreadPool threadPool, HttpSettings settings, ActionListener> listener) { - this.settings = Objects.requireNonNull(settings); + private volatile Exception exception; + + StreamingHttpResultPublisher(ThreadPool threadPool, HttpSettings settings, ActionListener listener) { this.listener = ActionListener.notifyOnce(Objects.requireNonNull(listener)); - this.taskRunner = new RequestBasedTaskRunner(new OffloadThread(), threadPool, UTILITY_THREAD_POOL_NAME); + this.publisher = new DataPublisher(threadPool); + this.backpressure = new ApacheClientBackpressure(Objects.requireNonNull(settings)); } @Override public void responseReceived(HttpResponse httpResponse) { - this.response = httpResponse; - } - - @Override - public void subscribe(Flow.Subscriber subscriber) { - if (this.subscriber != null) { - subscriber.onError(new IllegalStateException("Only one subscriber is allowed for this Publisher.")); - return; + if (listenerCalled.compareAndSet(false, true)) { + listener.onResponse(new StreamingHttpResult(httpResponse, publisher)); } - - this.subscriber = subscriber; - subscriber.onSubscribe(new HttpSubscription()); } @Override @@ -100,49 +81,20 @@ public void consumeContent(ContentDecoder contentDecoder, IOControl ioControl) t if (consumed > 0) { var allBytes = new byte[consumed]; inputBuffer.read(allBytes); - queue.offer(() -> { - subscriber.onNext(new HttpResult(response, allBytes)); - var currentBytesInQueue = bytesInQueue.updateAndGet(current -> Long.max(0, current - allBytes.length)); - if (savedIoControl != null) { - var maxBytes = settings.getMaxResponseSize().getBytes() * 0.5; - if (currentBytesInQueue <= maxBytes) { - resumeProducer(); - } - } - }); - - // always check if totalByteSize > the configured setting in case the settings change - if (bytesInQueue.accumulateAndGet(allBytes.length, Long::sum) >= settings.getMaxResponseSize().getBytes()) { - pauseProducer(ioControl); - } - - taskRunner.requestNextRun(); - - if (listenerCalled.compareAndSet(false, true)) { - listener.onResponse(this); - } + backpressure.addBytesAndMaybePause(consumed, ioControl); + publisher.onNext(allBytes); } + } catch (Exception e) { + // if the provider closes the connection in the middle of the stream, + // the contentDecoder will throw an exception trying to read the payload, + // we should catch that and forward it downstream so we can properly handle it + exception = e; + publisher.onError(e); } finally { inputBuffer.reset(); } } - private void pauseProducer(IOControl ioControl) { - ioControl.suspendInput(); - synchronized (ioLock) { - savedIoControl = ioControl; - } - } - - private void resumeProducer() { - synchronized (ioLock) { - if (savedIoControl != null) { - savedIoControl.requestInput(); - savedIoControl = null; - } - } - } - @Override public void responseCompleted(HttpContext httpContext) {} @@ -153,9 +105,8 @@ public void failed(Exception e) { if (listenerCalled.compareAndSet(false, true)) { listener.onFailure(e); } else { - ex = e; - queue.offer(() -> subscriber.onError(e)); - taskRunner.requestNextRun(); + exception = e; + publisher.onError(e); } } } @@ -164,8 +115,7 @@ public void failed(Exception e) { @Override public void close() { if (isDone.compareAndSet(false, true)) { - queue.offer(() -> subscriber.onComplete()); - taskRunner.requestNextRun(); + publisher.onComplete(); } } @@ -178,12 +128,12 @@ public boolean cancel() { @Override public Exception getException() { - return ex; + return exception; } @Override - public HttpResponse getResult() { - return response; + public Void getResult() { + return null; } @Override @@ -191,44 +141,148 @@ public boolean isDone() { return isDone.get(); } - private class HttpSubscription implements Flow.Subscription { + /** + * We only want to push payload data when the client is ready to receive it, so the client will use + * {@link Flow.Subscription#request(long)} to request more data. We collect the payload bytes in a queue and process them on a + * separate thread from both the Apache IO thread reading from the provider and the client's transport thread requesting more data. + * Clients use {@link Flow.Subscription#cancel()} to exit early, and we'll forward that cancellation to the provider. + */ + private class DataPublisher implements Flow.Processor { + private final RequestBasedTaskRunner taskRunner; + private final Deque contentQueue = new ConcurrentLinkedDeque<>(); + private final AtomicLong pendingRequests = new AtomicLong(0); + private volatile Exception pendingError = null; + private volatile boolean completed = false; + private volatile Flow.Subscriber downstream; + + private DataPublisher(ThreadPool threadPool) { + this.taskRunner = new RequestBasedTaskRunner(this::sendToSubscriber, threadPool, UTILITY_THREAD_POOL_NAME); + } + + private void sendToSubscriber() { + if (downstream == null) { + return; + } + if (pendingRequests.get() > 0 && pendingError != null) { + pendingRequests.decrementAndGet(); + downstream.onError(pendingError); + return; + } + byte[] nextBytes; + while (pendingRequests.get() > 0 && (nextBytes = contentQueue.poll()) != null) { + pendingRequests.decrementAndGet(); + backpressure.subtractBytesAndMaybeUnpause(nextBytes.length); + downstream.onNext(nextBytes); + } + if (pendingRequests.get() > 0 && contentQueue.isEmpty() && completed) { + pendingRequests.decrementAndGet(); + downstream.onComplete(); + } + } + @Override - public void request(long n) { - if (subscriptionCanceled.get()) { + public void subscribe(Flow.Subscriber subscriber) { + if (this.downstream != null) { + subscriber.onError(new IllegalStateException("Only one subscriber is allowed for this Publisher.")); return; } - if (n > 0) { - pendingRequest.set(true); - taskRunner.requestNextRun(); + this.downstream = subscriber; + downstream.onSubscribe(new Flow.Subscription() { + @Override + public void request(long n) { + if (n > 0) { + pendingRequests.addAndGet(n); + taskRunner.requestNextRun(); + } else { + // per Subscription's spec, fail the subscriber and stop the processor + cancel(); + subscriber.onError(new IllegalArgumentException("Subscriber requested a non-positive number " + n)); + } + } + + @Override + public void cancel() { + if (subscriptionCanceled.compareAndSet(false, true)) { + taskRunner.cancel(); + } + } + }); + } + + @Override + public void onNext(byte[] item) { + contentQueue.offer(item); + taskRunner.requestNextRun(); + } + + @Override + public void onError(Throwable throwable) { + if (throwable instanceof Exception e) { + pendingError = e; } else { - // per Subscription's spec, fail the subscriber and stop the processor - cancel(); - subscriber.onError(new IllegalArgumentException("Subscriber requested a non-positive number " + n)); + ExceptionsHelper.maybeError(throwable).ifPresent(ExceptionsHelper::maybeDieOnAnotherThread); + pendingError = new RuntimeException("Unhandled error while streaming"); } + taskRunner.requestNextRun(); } @Override - public void cancel() { - if (subscriptionCanceled.compareAndSet(false, true)) { - taskRunner.cancel(); - } + public void onComplete() { + completed = true; + taskRunner.requestNextRun(); } - } - private class OffloadThread implements Runnable { @Override - public void run() { - if (subscriptionCanceled.get()) { - return; + public void onSubscribe(Flow.Subscription subscription) { + assert false : "Apache never calls this."; + throw new UnsupportedOperationException("Apache never calls this."); + } + } + + /** + * We want to keep track of how much memory we are consuming while reading the payload from the external provider. Apache continuously + * pushes payload data to us, whereas the client only requests the next set of bytes when they are ready, so we want to track how much + * data we are holding in memory and potentially pause the Apache client if we have reached our limit. + */ + private static class ApacheClientBackpressure { + private final HttpSettings settings; + private final AtomicLong bytesInQueue = new AtomicLong(0); + private final Object ioLock = new Object(); + private volatile IOControl savedIoControl; + + private ApacheClientBackpressure(HttpSettings settings) { + this.settings = settings; + } + + private void addBytesAndMaybePause(long count, IOControl ioControl) { + if (bytesInQueue.addAndGet(count) >= settings.getMaxResponseSize().getBytes()) { + pauseProducer(ioControl); + } + } + + private void pauseProducer(IOControl ioControl) { + ioControl.suspendInput(); + synchronized (ioLock) { + savedIoControl = ioControl; + } + } + + private void subtractBytesAndMaybeUnpause(long count) { + var currentBytesInQueue = bytesInQueue.updateAndGet(current -> Long.max(0, current - count)); + if (savedIoControl != null) { + var maxBytes = settings.getMaxResponseSize().getBytes() * 0.5; + if (currentBytesInQueue <= maxBytes) { + resumeProducer(); + } } + } - if (queue.isEmpty() == false && pendingRequest.compareAndSet(true, false)) { - var next = queue.poll(); - if (next != null) { - next.run(); - } else { - pendingRequest.set(true); + private void resumeProducer() { + synchronized (ioLock) { + if (savedIoControl != null) { + savedIoControl.requestInput(); + savedIoControl = null; } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java index 1b0dd893ada6f..52a2ffba0c36c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java @@ -91,31 +91,24 @@ protected Exception buildError(String message, Request request, HttpResult resul protected Exception buildError(String message, Request request, HttpResult result, ErrorResponse errorResponse) { var responseStatusCode = result.response().getStatusLine().getStatusCode(); + return new ElasticsearchStatusException( + errorMessage(message, request, result, errorResponse, responseStatusCode), + toRestStatus(responseStatusCode) + ); + } - if (errorResponse == null + protected String errorMessage(String message, Request request, HttpResult result, ErrorResponse errorResponse, int statusCode) { + return (errorResponse == null || errorResponse.errorStructureFound() == false - || Strings.isNullOrEmpty(errorResponse.getErrorMessage())) { - return new ElasticsearchStatusException( - format( - "%s for request from inference entity id [%s] status [%s]", + || Strings.isNullOrEmpty(errorResponse.getErrorMessage())) + ? format("%s for request from inference entity id [%s] status [%s]", message, request.getInferenceEntityId(), statusCode) + : format( + "%s for request from inference entity id [%s] status [%s]. Error message: [%s]", message, request.getInferenceEntityId(), - responseStatusCode - ), - toRestStatus(responseStatusCode) - ); - } - - return new ElasticsearchStatusException( - format( - "%s for request from inference entity id [%s] status [%s]. Error message: [%s]", - message, - request.getInferenceEntityId(), - responseStatusCode, - errorResponse.getErrorMessage() - ), - toRestStatus(responseStatusCode) - ); + statusCode, + errorResponse.getErrorMessage() + ); } public static RestStatus toRestStatus(int statusCode) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java index 1c303f6e965cf..b71887ce6018f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java @@ -116,9 +116,20 @@ public void tryAction(ActionListener listener) { try { if (request.isStreaming() && responseHandler.canHandleStreamingResponses()) { httpClient.stream(request.createHttpRequest(), context, retryableListener.delegateFailure((l, r) -> { - var streamingResponseHandler = new StreamingResponseHandler(throttlerManager, logger, request, responseHandler); - r.subscribe(streamingResponseHandler); - l.onResponse(responseHandler.parseResult(request, streamingResponseHandler)); + if (r.isSuccessfulResponse()) { + l.onResponse(responseHandler.parseResult(request, r.toHttpResult())); + } else { + r.readFullResponse(l.delegateFailureAndWrap((ll, httpResult) -> { + try { + responseHandler.validateResponse(throttlerManager, logger, request, httpResult); + InferenceServiceResults inferenceResults = responseHandler.parseResult(request, httpResult); + ll.onResponse(inferenceResults); + } catch (Exception e) { + logException(logger, request, httpResult, responseHandler.getRequestType(), e); + listener.onFailure(e); // skip retrying + } + })); + } })); } else { httpClient.send(request.createHttpRequest(), context, retryableListener.delegateFailure((l, r) -> { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/StreamingResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/StreamingResponseHandler.java deleted file mode 100644 index 44e04ae28751f..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/StreamingResponseHandler.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.http.retry; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.request.Request; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; - -import java.util.concurrent.Flow; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.elasticsearch.core.Strings.format; - -class StreamingResponseHandler implements Flow.Processor { - private static final Logger log = LogManager.getLogger(StreamingResponseHandler.class); - private final ThrottlerManager throttlerManager; - private final Logger throttlerLogger; - private final Request request; - private final ResponseHandler responseHandler; - - private final AtomicBoolean upstreamIsClosed = new AtomicBoolean(false); - private final AtomicBoolean processedFirstItem = new AtomicBoolean(false); - - private volatile Flow.Subscription upstream; - private volatile Flow.Subscriber downstream; - - StreamingResponseHandler(ThrottlerManager throttlerManager, Logger throttlerLogger, Request request, ResponseHandler responseHandler) { - this.throttlerManager = throttlerManager; - this.throttlerLogger = throttlerLogger; - this.request = request; - this.responseHandler = responseHandler; - } - - @Override - public void subscribe(Flow.Subscriber subscriber) { - if (downstream != null) { - subscriber.onError( - new IllegalStateException("Failed to initialize streaming response. Another subscriber is already subscribed.") - ); - return; - } - - downstream = subscriber; - subscriber.onSubscribe(forwardingSubscription()); - } - - private Flow.Subscription forwardingSubscription() { - return new Flow.Subscription() { - @Override - public void request(long n) { - if (upstreamIsClosed.get()) { - downstream.onComplete(); // shouldn't happen, but reinforce that we're no longer listening - } else if (upstream != null) { - upstream.request(n); - } else { - // this shouldn't happen, the expected call pattern is onNext -> subscribe after the listener is invoked - var errorMessage = "Failed to initialize streaming response. onSubscribe must be called first to set the upstream"; - assert false : errorMessage; - downstream.onError(new IllegalStateException(errorMessage)); - } - } - - @Override - public void cancel() { - if (upstreamIsClosed.compareAndSet(false, true) && upstream != null) { - upstream.cancel(); - } - } - }; - } - - @Override - public void onSubscribe(Flow.Subscription subscription) { - upstream = subscription; - } - - @Override - public void onNext(HttpResult item) { - if (processedFirstItem.compareAndSet(false, true)) { - try { - responseHandler.validateResponse(throttlerManager, throttlerLogger, request, item); - } catch (Exception e) { - logException(throttlerLogger, request, item, responseHandler.getRequestType(), e); - upstream.cancel(); - onError(e); - return; - } - } - downstream.onNext(item); - } - - @Override - public void onError(Throwable throwable) { - if (upstreamIsClosed.compareAndSet(false, true)) { - if (downstream != null) { - downstream.onError(throwable); - } else { - log.warn( - "Flow failed before the InferenceServiceResults were generated. The error should go to the listener directly.", - throwable - ); - } - } - } - - @Override - public void onComplete() { - if (upstreamIsClosed.compareAndSet(false, true)) { - if (downstream != null) { - downstream.onComplete(); - } else { - log.debug("Flow completed before the InferenceServiceResults were generated. Shutting down this Processor."); - } - } - } - - private void logException(Logger logger, Request request, HttpResult result, String requestType, Exception exception) { - var causeException = ExceptionsHelper.unwrapCause(exception); - - throttlerManager.warn( - logger, - format( - "Failed to process the stream connection for request from inference entity id [%s] of type [%s] with status [%s] [%s]", - request.getInferenceEntityId(), - requestType, - result.response().getStatusLine().getStatusCode(), - result.response().getStatusLine().getReasonPhrase() - ), - causeException - ); - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java index 928da95d9c2f0..58c952b9c556a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.inference.external.http.sender; +import org.elasticsearch.inference.TaskType; + import java.util.List; import java.util.Objects; @@ -15,7 +17,7 @@ * The main difference between this class and {@link UnifiedChatInput} is this should only be used for * {@link org.elasticsearch.inference.TaskType#COMPLETION} originating through the * {@link org.elasticsearch.inference.InferenceService#infer} code path. These are requests sent to the - * API without using the _unified route. + * API without using the {@link TaskType#CHAT_COMPLETION} task type. */ public class ChatCompletionInput extends InferenceInputs { private final List input; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/IbmWatsonxRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/IbmWatsonxRerankRequestManager.java new file mode 100644 index 0000000000000..f503771510e72 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/IbmWatsonxRerankRequestManager.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.ibmwatsonx.IbmWatsonxResponseHandler; +import org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxRerankRequest; +import org.elasticsearch.xpack.inference.external.response.ibmwatsonx.IbmWatsonxRankedResponseEntity; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; + +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +public class IbmWatsonxRerankRequestManager extends IbmWatsonxRequestManager { + private static final Logger logger = LogManager.getLogger(IbmWatsonxRerankRequestManager.class); + private static final ResponseHandler HANDLER = createIbmWatsonxResponseHandler(); + + private static ResponseHandler createIbmWatsonxResponseHandler() { + return new IbmWatsonxResponseHandler( + "ibm watsonx rerank", + (request, response) -> IbmWatsonxRankedResponseEntity.fromResponse(response) + ); + } + + public static IbmWatsonxRerankRequestManager of(IbmWatsonxRerankModel model, ThreadPool threadPool) { + return new IbmWatsonxRerankRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + + private final IbmWatsonxRerankModel model; + + public IbmWatsonxRerankRequestManager(IbmWatsonxRerankModel model, ThreadPool threadPool) { + super(threadPool, model); + this.model = model; + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + var rerankInput = QueryAndDocsInputs.of(inferenceInputs); + + execute( + new ExecutableInferenceRequest( + requestSender, + logger, + getRerankRequest(rerankInput.getQuery(), rerankInput.getChunks(), model), + HANDLER, + hasRequestCompletedFunction, + listener + ) + ); + } + + protected IbmWatsonxRerankRequest getRerankRequest(String query, List chunks, IbmWatsonxRerankModel model) { + return new IbmWatsonxRerankRequest(query, chunks, model); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java index fceec7c431182..f4f0511a4cc1b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; import java.util.List; @@ -20,7 +21,7 @@ * The main difference between this class and {@link ChatCompletionInput} is this should only be used for * {@link org.elasticsearch.inference.TaskType#COMPLETION} originating through the * {@link org.elasticsearch.inference.InferenceService#unifiedCompletionInfer(Model, UnifiedCompletionRequest, TimeValue, ActionListener)} - * code path. These are requests sent to the API with the _unified route. + * code path. These are requests sent to the API with the _stream route and {@link TaskType#CHAT_COMPLETION}. */ public class UnifiedChatInput extends InferenceInputs { private final UnifiedCompletionRequest request; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java index 019d72a381c27..f5512c311c5d6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java @@ -17,7 +17,6 @@ import static org.elasticsearch.core.Strings.format; public class IbmWatsonxResponseHandler extends BaseResponseHandler { - public IbmWatsonxResponseHandler(String requestType, ResponseParser parseFunction) { super(requestType, parseFunction, IbmWatsonxErrorResponseEntity::fromResponse); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java index 7607e5e4ed3a2..99f2a7c31e7dd 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java @@ -8,15 +8,26 @@ package org.elasticsearch.xpack.inference.external.openai; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; +import java.util.function.Function; + public class OpenAiChatCompletionResponseHandler extends OpenAiResponseHandler { public OpenAiChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { super(requestType, parseFunction, true); } + protected OpenAiChatCompletionResponseHandler( + String requestType, + ResponseParser parseFunction, + Function errorParseFunction + ) { + super(requestType, parseFunction, errorParseFunction, true); + } + @Override protected RetryException buildExceptionHandling429(Request request, HttpResult result) { // We don't retry, if the chat completion input is too large diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java index f75b89dca939f..33aabf42b9583 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; @@ -21,6 +22,7 @@ import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; import java.util.concurrent.Flow; +import java.util.function.Function; import static org.elasticsearch.xpack.inference.external.http.retry.ResponseHandlerUtils.getFirstHeaderOrUnknown; @@ -44,7 +46,16 @@ public class OpenAiResponseHandler extends BaseResponseHandler { private final boolean canHandleStreamingResponses; public OpenAiResponseHandler(String requestType, ResponseParser parseFunction, boolean canHandleStreamingResponses) { - super(requestType, parseFunction, ErrorMessageResponseEntity::fromResponse); + this(requestType, parseFunction, ErrorMessageResponseEntity::fromResponse, canHandleStreamingResponses); + } + + protected OpenAiResponseHandler( + String requestType, + ResponseParser parseFunction, + Function errorParseFunction, + boolean canHandleStreamingResponses + ) { + super(requestType, parseFunction, errorParseFunction); this.canHandleStreamingResponses = canHandleStreamingResponses; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java index fce2556efc5e0..b2096253bdeb7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java @@ -7,28 +7,171 @@ package org.elasticsearch.xpack.inference.external.openai; +import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.Flow; +import static org.elasticsearch.core.Strings.format; + public class OpenAiUnifiedChatCompletionResponseHandler extends OpenAiChatCompletionResponseHandler { public OpenAiUnifiedChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { - super(requestType, parseFunction); + super(requestType, parseFunction, OpenAiErrorResponse::fromResponse); } @Override public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); - var openAiProcessor = new OpenAiUnifiedStreamingProcessor(); + var openAiProcessor = new OpenAiUnifiedStreamingProcessor((m, e) -> buildMidStreamError(request, m, e)); flow.subscribe(serverSentEventProcessor); serverSentEventProcessor.subscribe(openAiProcessor); return new StreamingUnifiedChatCompletionResults(openAiProcessor); } + + @Override + protected Exception buildError(String message, Request request, HttpResult result, ErrorResponse errorResponse) { + assert request.isStreaming() : "Only streaming requests support this format"; + var responseStatusCode = result.response().getStatusLine().getStatusCode(); + if (request.isStreaming()) { + var errorMessage = errorMessage(message, request, result, errorResponse, responseStatusCode); + var restStatus = toRestStatus(responseStatusCode); + return errorResponse instanceof OpenAiErrorResponse oer + ? new UnifiedChatCompletionException(restStatus, errorMessage, oer.type(), oer.code(), oer.param()) + : new UnifiedChatCompletionException( + restStatus, + errorMessage, + errorResponse != null ? errorResponse.getClass().getSimpleName() : "unknown", + restStatus.name().toLowerCase(Locale.ROOT) + ); + } else { + return super.buildError(message, request, result, errorResponse); + } + } + + private static Exception buildMidStreamError(Request request, String message, Exception e) { + var errorResponse = OpenAiErrorResponse.fromString(message); + if (errorResponse instanceof OpenAiErrorResponse oer) { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format( + "%s for request from inference entity id [%s]. Error message: [%s]", + SERVER_ERROR_OBJECT, + request.getInferenceEntityId(), + errorResponse.getErrorMessage() + ), + oer.type(), + oer.code(), + oer.param() + ); + } else if (e != null) { + return UnifiedChatCompletionException.fromThrowable(e); + } else { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format("%s for request from inference entity id [%s]", SERVER_ERROR_OBJECT, request.getInferenceEntityId()), + errorResponse != null ? errorResponse.getClass().getSimpleName() : "unknown", + "stream_error" + ); + } + } + + private static class OpenAiErrorResponse extends ErrorResponse { + private static final ConstructingObjectParser, Void> ERROR_PARSER = new ConstructingObjectParser<>( + "open_ai_error", + true, + args -> Optional.ofNullable((OpenAiErrorResponse) args[0]) + ); + private static final ConstructingObjectParser ERROR_BODY_PARSER = new ConstructingObjectParser<>( + "open_ai_error", + true, + args -> new OpenAiErrorResponse((String) args[0], (String) args[1], (String) args[2], (String) args[3]) + ); + + static { + ERROR_BODY_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("message")); + ERROR_BODY_PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("code")); + ERROR_BODY_PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("param")); + ERROR_BODY_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("type")); + + ERROR_PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + ERROR_BODY_PARSER, + null, + new ParseField("error") + ); + } + + private static ErrorResponse fromResponse(HttpResult response) { + try ( + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, response.body()) + ) { + return ERROR_PARSER.apply(parser, null).orElse(ErrorResponse.UNDEFINED_ERROR); + } catch (Exception e) { + // swallow the error + } + + return ErrorResponse.UNDEFINED_ERROR; + } + + private static ErrorResponse fromString(String response) { + try ( + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, response) + ) { + return ERROR_PARSER.apply(parser, null).orElse(ErrorResponse.UNDEFINED_ERROR); + } catch (Exception e) { + // swallow the error + } + + return ErrorResponse.UNDEFINED_ERROR; + } + + @Nullable + private final String code; + @Nullable + private final String param; + private final String type; + + OpenAiErrorResponse(String errorMessage, @Nullable String code, @Nullable String param, String type) { + super(errorMessage); + this.code = code; + this.param = param; + this.type = Objects.requireNonNull(type); + } + + @Nullable + public String code() { + return code; + } + + @Nullable + public String param() { + return param; + } + + public String type() { + return type; + } + } + } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java index 599d71df3dcfa..bfd4456279a8a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; import org.elasticsearch.xpack.inference.common.DelegatingProcessor; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; import java.io.IOException; import java.util.ArrayDeque; @@ -28,6 +29,7 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingDeque; +import java.util.function.BiFunction; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; @@ -57,7 +59,13 @@ public class OpenAiUnifiedStreamingProcessor extends DelegatingProcessor errorParser; private final Deque buffer = new LinkedBlockingDeque<>(); + private volatile boolean previousEventWasError = false; + + public OpenAiUnifiedStreamingProcessor(BiFunction errorParser) { + this.errorParser = errorParser; + } @Override protected void upstreamRequest(long n) { @@ -71,7 +79,25 @@ protected void upstreamRequest(long n) { @Override protected void next(Deque item) throws Exception { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); - var results = parseEvent(item, OpenAiUnifiedStreamingProcessor::parse, parserConfig, logger); + + var results = new ArrayDeque(item.size()); + for (var event : item) { + if (ServerSentEventField.EVENT == event.name() && "error".equals(event.value())) { + previousEventWasError = true; + } else if (ServerSentEventField.DATA == event.name() && event.hasValue()) { + if (previousEventWasError) { + throw errorParser.apply(event.value(), null); + } + + try { + var delta = parse(parserConfig, event); + delta.forEachRemaining(results::offer); + } catch (Exception e) { + logger.warn("Failed to parse event from inference provider: {}", event); + throw errorParser.apply(event.value(), e); + } + } + } if (results.isEmpty()) { upstream().request(1); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java index cba5447f8ff27..0ba6b46da05e5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java @@ -23,9 +23,7 @@ public record ElasticInferenceServiceSparseEmbeddingsRequestEntity( ) implements ToXContentObject { private static final String INPUT_FIELD = "input"; - - private static final String MODEL_ID_FIELD = "model_id"; - + private static final String MODEL_FIELD = "model"; private static final String USAGE_CONTEXT = "usage_context"; public ElasticInferenceServiceSparseEmbeddingsRequestEntity { @@ -44,7 +42,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endArray(); - builder.field(MODEL_ID_FIELD, modelId); + builder.field(MODEL_FIELD, modelId); // optional field if ((usageContext == ElasticInferenceServiceUsageContext.UNSPECIFIED) == false) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRerankRequest.java new file mode 100644 index 0000000000000..cfc1f367be45c --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRerankRequest.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.ibmwatsonx; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankTaskSettings; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; + +public class IbmWatsonxRerankRequest implements IbmWatsonxRequest { + + private final String query; + private final List input; + private final IbmWatsonxRerankTaskSettings taskSettings; + private final IbmWatsonxRerankModel model; + + public IbmWatsonxRerankRequest(String query, List input, IbmWatsonxRerankModel model) { + Objects.requireNonNull(model); + + this.input = Objects.requireNonNull(input); + this.query = Objects.requireNonNull(query); + taskSettings = model.getTaskSettings(); + this.model = model; + } + + @Override + public HttpRequest createHttpRequest() { + URI uri; + + try { + uri = new URI(model.uri().toString()); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("cannot parse URI patter"); + } + + HttpPost httpPost = new HttpPost(uri); + + ByteArrayEntity byteEntity = new ByteArrayEntity( + Strings.toString( + new IbmWatsonxRerankRequestEntity( + query, + input, + taskSettings, + model.getServiceSettings().modelId(), + model.getServiceSettings().projectId() + ) + ).getBytes(StandardCharsets.UTF_8) + ); + + httpPost.setEntity(byteEntity); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()); + + decorateWithAuth(httpPost); + + return new HttpRequest(httpPost, getInferenceEntityId()); + } + + public void decorateWithAuth(HttpPost httpPost) { + IbmWatsonxRequest.decorateWithBearerToken(httpPost, model.getSecretSettings(), model.getInferenceEntityId()); + } + + @Override + public String getInferenceEntityId() { + return model.getInferenceEntityId(); + } + + @Override + public URI getURI() { + return model.uri(); + } + + @Override + public Request truncate() { + return this; + } + + public String getQuery() { + return query; + } + + public List getInput() { + return input; + } + + public IbmWatsonxRerankModel getModel() { + return model; + } + + @Override + public boolean[] getTruncationInfo() { + return null; + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRerankRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRerankRequestEntity.java new file mode 100644 index 0000000000000..36e5951ebdc15 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRerankRequestEntity.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.ibmwatsonx; + +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankTaskSettings; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public record IbmWatsonxRerankRequestEntity( + String query, + List inputs, + IbmWatsonxRerankTaskSettings taskSettings, + String modelId, + String projectId +) implements ToXContentObject { + + private static final String INPUTS_FIELD = "inputs"; + private static final String QUERY_FIELD = "query"; + private static final String MODEL_ID_FIELD = "model_id"; + private static final String PROJECT_ID_FIELD = "project_id"; + + public IbmWatsonxRerankRequestEntity { + Objects.requireNonNull(query); + Objects.requireNonNull(inputs); + Objects.requireNonNull(modelId); + Objects.requireNonNull(projectId); + Objects.requireNonNull(taskSettings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.field(MODEL_ID_FIELD, modelId); + builder.field(QUERY_FIELD, query); + builder.startArray(INPUTS_FIELD); + for (String input : inputs) { + builder.startObject(); + builder.field("text", input); + builder.endObject(); + } + builder.endArray(); + builder.field(PROJECT_ID_FIELD, projectId); + + builder.startObject("parameters"); + { + if (taskSettings.getTruncateInputTokens() != null) { + builder.field("truncate_input_tokens", taskSettings.getTruncateInputTokens()); + } + + builder.startObject("return_options"); + { + if (taskSettings.getDoesReturnDocuments() != null) { + builder.field("inputs", taskSettings.getDoesReturnDocuments()); + } + if (taskSettings.getTopNDocumentsOnly() != null) { + builder.field("top_n", taskSettings.getTopNDocumentsOnly()); + } + } + builder.endObject(); + } + builder.endObject(); + + builder.endObject(); + + return builder; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxUtils.java index a506a33385dfb..91679288e5ae3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxUtils.java @@ -13,6 +13,7 @@ public class IbmWatsonxUtils { public static final String V1 = "v1"; public static final String TEXT = "text"; public static final String EMBEDDINGS = "embeddings"; + public static final String RERANKS = "reranks"; private IbmWatsonxUtils() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java index 696be7b2acdd2..29b0903901694 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -16,6 +17,18 @@ import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; +import java.io.IOException; + +/** + * An example error response would look like + * + * + * { + * "error": "some error" + * } + * + * + */ public class ElasticInferenceServiceErrorResponseEntity extends ErrorResponse { private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceErrorResponseEntity.class); @@ -24,24 +37,18 @@ private ElasticInferenceServiceErrorResponseEntity(String errorMessage) { super(errorMessage); } - /** - * An example error response would look like - * - * - * { - * "error": "some error" - * } - * - * - * @param response The error response - * @return An error entity if the response is JSON with the above structure - * or {@link ErrorResponse#UNDEFINED_ERROR} if the error field wasn't found - */ public static ErrorResponse fromResponse(HttpResult response) { - try ( - XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) - .createParser(XContentParserConfiguration.EMPTY, response.body()) - ) { + return fromParser( + () -> XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, response.body()) + ); + } + + public static ErrorResponse fromString(String response) { + return fromParser(() -> XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, response)); + } + + private static ErrorResponse fromParser(CheckedSupplier jsonParserFactory) { + try (XContentParser jsonParser = jsonParserFactory.get()) { var responseMap = jsonParser.map(); var error = (String) responseMap.get("error"); if (error != null) { @@ -50,7 +57,6 @@ public static ErrorResponse fromResponse(HttpResult response) { } catch (Exception e) { logger.debug("Failed to parse error response", e); } - return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxRankedResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxRankedResponseEntity.java new file mode 100644 index 0000000000000..05f369bd8961e --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxRankedResponseEntity.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.inference.external.response.ibmwatsonx; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList; +import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField; +import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField; + +public class IbmWatsonxRankedResponseEntity { + + private static final Logger logger = LogManager.getLogger(IbmWatsonxRankedResponseEntity.class); + + /** + * Parses the Ibm Watsonx ranked response. + * + * For a request like: + * "model": "rerank-english-v2.0", + * "query": "database", + * "return_documents": true, + * "top_n": 3, + * "input": ["greenland", "google","john", "mysql","potter", "grammar"] + *

+ * The response will look like (without whitespace): + * { + * "rerank": [ + * { + * "index": 3, + * "relevance_score": 0.7989932 + * }, + * { + * "index": 5, + * "relevance_score": 0.61281824 + * }, + * { + * "index": 1, + * "relevance_score": 0.5762553 + * }, + * { + * "index": 4, + * "relevance_score": 0.47395563 + * }, + * { + * "index": 0, + * "relevance_score": 0.4338926 + * }, + * { + * "index": 2, + * "relevance_score": 0.42638257 + * } + * ], + * } + * + * @param response the http response from ibm watsonx + * @return the parsed response + * @throws IOException if there is an error parsing the response + */ + public static InferenceServiceResults fromResponse(HttpResult response) throws IOException { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { + moveToFirstToken(jsonParser); + + XContentParser.Token token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + + positionParserAtTokenAfterField(jsonParser, "results", FAILED_TO_FIND_FIELD_TEMPLATE); // TODO error message + + token = jsonParser.currentToken(); + if (token == XContentParser.Token.START_ARRAY) { + return new RankedDocsResults(parseList(jsonParser, IbmWatsonxRankedResponseEntity::parseRankedDocObject)); + } else { + throwUnknownToken(token, jsonParser); + } + + // This should never be reached. The above code should either return successfully or hit the throwUnknownToken + // or throw a parsing exception + throw new IllegalStateException("Reached an invalid state while parsing the Watsonx response"); + } + } + + private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + int index = -1; + float score = -1; + String documentText = null; + parser.nextToken(); + while (parser.currentToken() != XContentParser.Token.END_OBJECT) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + switch (parser.currentName()) { + case "index": + parser.nextToken(); // move to VALUE_NUMBER + index = parser.intValue(); + parser.nextToken(); // move to next FIELD_NAME or END_OBJECT + break; + case "score": + parser.nextToken(); // move to VALUE_NUMBER + score = parser.floatValue(); + parser.nextToken(); // move to next FIELD_NAME or END_OBJECT + break; + case "input": + parser.nextToken(); // move to START_OBJECT; document text is wrapped in an object + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + do { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME && parser.currentName().equals("text")) { + parser.nextToken(); // move to VALUE_STRING + documentText = parser.text(); + } + } while (parser.nextToken() != XContentParser.Token.END_OBJECT); + parser.nextToken();// move past END_OBJECT + // parser should now be at the next FIELD_NAME or END_OBJECT + break; + default: + throwUnknownField(parser.currentName(), parser); + } + } else { + parser.nextToken(); + } + } + + if (index == -1) { + logger.warn("Failed to find required field [index] in Watsonx rerank response"); + } + if (score == -1) { + logger.warn("Failed to find required field [relevance_score] in Watsonx rerank response"); + } + // documentText may or may not be present depending on the request parameter + + return new RankedDocsResults.RankedDoc(index, score, documentText); + } + + private IbmWatsonxRankedResponseEntity() {} + + static String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in Watsonx rerank response"; +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 9c2ad5919722f..700194ddc6b58 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -119,6 +119,7 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie public static final NodeFeature SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX = new NodeFeature( "semantic_text.always_emit_inference_id_fix" ); + public static final NodeFeature SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS = new NodeFeature("semantic_text.skip_inference_fields"); public static final String CONTENT_TYPE = "semantic_text"; public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java index 57805d5277ffc..9e513a1ed9226 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticKnnVectorQueryRewriteInterceptor.java @@ -28,6 +28,7 @@ public class SemanticKnnVectorQueryRewriteInterceptor extends SemanticQueryRewri public static final NodeFeature SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED = new NodeFeature( "search.semantic_knn_vector_query_rewrite_interception_supported" ); + public static final NodeFeature SEMANTIC_KNN_FILTER_FIX = new NodeFeature("search.semantic_knn_filter_fix"); public SemanticKnnVectorQueryRewriteInterceptor() {} @@ -147,6 +148,7 @@ private KnnVectorQueryBuilder addIndexFilterToKnnVectorQuery(Collection ); } + copy.addFilterQueries(original.filterQueries()); copy.addFilterQuery(new TermsQueryBuilder(IndexFieldMapper.NAME, indices)); return copy; } @@ -165,8 +167,9 @@ private KnnVectorQueryBuilder buildNewKnnVectorQuery( KnnVectorQueryBuilder original, QueryVectorBuilder queryVectorBuilder ) { + KnnVectorQueryBuilder newQueryBuilder; if (original.queryVectorBuilder() != null) { - return new KnnVectorQueryBuilder( + newQueryBuilder = new KnnVectorQueryBuilder( fieldName, queryVectorBuilder, original.k(), @@ -174,7 +177,7 @@ private KnnVectorQueryBuilder buildNewKnnVectorQuery( original.getVectorSimilarity() ); } else { - return new KnnVectorQueryBuilder( + newQueryBuilder = new KnnVectorQueryBuilder( fieldName, original.queryVector(), original.k(), @@ -183,6 +186,9 @@ private KnnVectorQueryBuilder buildNewKnnVectorQuery( original.getVectorSimilarity() ); } + + newQueryBuilder.addFilterQueries(original.filterQueries()); + return newQueryBuilder; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java index a9642a685aec9..ca7595f78da06 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.engine.VersionConflictEngineException; @@ -61,6 +62,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -111,7 +113,7 @@ public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) public ModelRegistry(Client client) { this.client = new OriginSettingClient(client, ClientHelper.INFERENCE_ORIGIN); - defaultConfigIds = new HashMap<>(); + defaultConfigIds = new ConcurrentHashMap<>(); } /** @@ -124,11 +126,20 @@ public boolean containsDefaultConfigId(String inferenceEntityId) { return defaultConfigIds.containsKey(inferenceEntityId); } + /** + * Adds the default configuration information if it does not already exist internally. + * @param defaultConfigId the default endpoint information + */ + public synchronized void putDefaultIdIfAbsent(InferenceService.DefaultConfigId defaultConfigId) { + defaultConfigIds.putIfAbsent(defaultConfigId.inferenceId(), defaultConfigId); + } + /** * Set the default inference ids provided by the services - * @param defaultConfigId The default + * @param defaultConfigId The default endpoint information + * @throws IllegalStateException if the {@link InferenceService.DefaultConfigId#inferenceId()} already exists internally */ - public synchronized void addDefaultIds(InferenceService.DefaultConfigId defaultConfigId) { + public synchronized void addDefaultIds(InferenceService.DefaultConfigId defaultConfigId) throws IllegalStateException { var config = defaultConfigIds.get(defaultConfigId.inferenceId()); if (config != null) { throw new IllegalStateException( @@ -644,11 +655,32 @@ private static BulkItemResponse.Failure getFirstBulkFailure(BulkResponse bulkRes return null; } + public synchronized void removeDefaultConfigs(Set inferenceEntityIds, ActionListener listener) { + if (inferenceEntityIds.isEmpty()) { + listener.onResponse(true); + return; + } + + defaultConfigIds.keySet().removeAll(inferenceEntityIds); + deleteModels(inferenceEntityIds, listener); + } + public void deleteModel(String inferenceEntityId, ActionListener listener) { - if (preventDeletionLock.contains(inferenceEntityId)) { + deleteModels(Set.of(inferenceEntityId), listener); + } + + public void deleteModels(Set inferenceEntityIds, ActionListener listener) { + var lockedInferenceIds = new HashSet<>(inferenceEntityIds); + lockedInferenceIds.retainAll(preventDeletionLock); + + if (lockedInferenceIds.isEmpty() == false) { listener.onFailure( new ElasticsearchStatusException( - "Model is currently being updated, you may delete the model once the update completes", + Strings.format( + "The inference endpoint(s) %s are currently being updated, please wait until after they are " + + "finished updating to delete.", + lockedInferenceIds + ), RestStatus.CONFLICT ) ); @@ -657,7 +689,7 @@ public void deleteModel(String inferenceEntityId, ActionListener listen DeleteByQueryRequest request = new DeleteByQueryRequest().setAbortOnVersionConflict(false); request.indices(InferenceIndex.INDEX_PATTERN, InferenceSecretsIndex.INDEX_PATTERN); - request.setQuery(documentIdQuery(inferenceEntityId)); + request.setQuery(documentIdsQuery(inferenceEntityIds)); request.setRefresh(true); client.execute(DeleteByQueryAction.INSTANCE, request, listener.delegateFailureAndWrap((l, r) -> l.onResponse(Boolean.TRUE))); @@ -695,6 +727,11 @@ private QueryBuilder documentIdQuery(String inferenceEntityId) { return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId))); } + private QueryBuilder documentIdsQuery(Set inferenceEntityIds) { + var documentIdsArray = inferenceEntityIds.stream().map(Model::documentId).toArray(String[]::new); + return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(documentIdsArray)); + } + static Optional idMatchedDefault( String inferenceId, List defaultConfigIds diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java index d911158e82296..06a0849b91d4e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; import java.io.IOException; @@ -41,21 +42,22 @@ static TimeValue parseTimeout(RestRequest restRequest) { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { var params = parseParams(restRequest); + var content = restRequest.requiredContent(); + var inferTimeout = parseTimeout(restRequest); - InferenceAction.Request.Builder requestBuilder; - try (var parser = restRequest.contentParser()) { - requestBuilder = InferenceAction.Request.parseRequest(params.inferenceEntityId(), params.taskType(), parser); - } + var request = new InferenceActionProxy.Request( + params.taskType(), + params.inferenceEntityId(), + content, + restRequest.getXContentType(), + inferTimeout, + shouldStream() + ); - var inferTimeout = parseTimeout(restRequest); - requestBuilder.setInferenceTimeout(inferTimeout); - var request = prepareInferenceRequest(requestBuilder); - return channel -> client.execute(InferenceAction.INSTANCE, request, listener(channel)); + return channel -> client.execute(InferenceActionProxy.INSTANCE, request, ActionListener.withRef(listener(channel), content)); } - protected InferenceAction.Request prepareInferenceRequest(InferenceAction.Request.Builder builder) { - return builder.build(); - } + protected abstract boolean shouldStream(); protected abstract ActionListener listener(RestChannel channel); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java index 57c06df8d8dfe..b1edec79dfb72 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java @@ -24,21 +24,14 @@ public final class Paths { static final String INFERENCE_SERVICES_PATH = "_inference/_services"; static final String TASK_TYPE_INFERENCE_SERVICES_PATH = "_inference/_services/{" + TASK_TYPE + "}"; - static final String STREAM_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/_stream"; + public static final String STREAM_SUFFIX = "_stream"; + static final String STREAM_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/" + STREAM_SUFFIX; static final String STREAM_TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" - + TASK_TYPE_OR_INFERENCE_ID - + "}/{" - + INFERENCE_ID - + "}/_stream"; - - public static final String UNIFIED_SUFFIX = "_unified"; - static final String UNIFIED_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/" + UNIFIED_SUFFIX; - static final String UNIFIED_TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/{" + INFERENCE_ID + "}/" - + UNIFIED_SUFFIX; + + STREAM_SUFFIX; private Paths() { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java index 0fbc2f8214cbb..55083dcd4c888 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java @@ -32,6 +32,11 @@ public List routes() { return List.of(new Route(POST, INFERENCE_ID_PATH), new Route(POST, TASK_TYPE_INFERENCE_ID_PATH)); } + @Override + protected boolean shouldStream() { + return false; + } + @Override protected ActionListener listener(RestChannel channel) { return new RestChunkedToXContentListener<>(channel); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java index 881af435b29b6..f37f4e9fb1f9b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceAction.java @@ -42,12 +42,12 @@ public List routes() { } @Override - protected InferenceAction.Request prepareInferenceRequest(InferenceAction.Request.Builder builder) { - return builder.setStream(true).build(); + protected ActionListener listener(RestChannel channel) { + return new ServerSentEventsRestActionListener(channel, threadPool); } @Override - protected ActionListener listener(RestChannel channel) { - return new ServerSentEventsRestActionListener(channel, threadPool); + protected boolean shouldStream() { + return true; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java deleted file mode 100644 index 51f1bc48c8306..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.rest; - -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.Scope; -import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.xpack.inference.rest.Paths.UNIFIED_INFERENCE_ID_PATH; -import static org.elasticsearch.xpack.inference.rest.Paths.UNIFIED_TASK_TYPE_INFERENCE_ID_PATH; - -@ServerlessScope(Scope.PUBLIC) -public class RestUnifiedCompletionInferenceAction extends BaseRestHandler { - private final SetOnce threadPool; - - public RestUnifiedCompletionInferenceAction(SetOnce threadPool) { - super(); - this.threadPool = Objects.requireNonNull(threadPool); - } - - @Override - public String getName() { - return "unified_inference_action"; - } - - @Override - public List routes() { - return List.of(new Route(POST, UNIFIED_INFERENCE_ID_PATH), new Route(POST, UNIFIED_TASK_TYPE_INFERENCE_ID_PATH)); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - var params = BaseInferenceAction.parseParams(restRequest); - - var inferTimeout = BaseInferenceAction.parseTimeout(restRequest); - - UnifiedCompletionAction.Request request; - try (var parser = restRequest.contentParser()) { - request = UnifiedCompletionAction.Request.parseRequest(params.inferenceEntityId(), params.taskType(), inferTimeout, parser); - } - - return channel -> client.execute( - UnifiedCompletionAction.INSTANCE, - request, - new ServerSentEventsRestActionListener(channel, threadPool) - ); - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java index 120731a4f8e66..7b3c54c60cdcc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java @@ -7,13 +7,11 @@ package org.elasticsearch.xpack.inference.rest; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; @@ -48,7 +46,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient inferenceEntityId = restRequest.param(INFERENCE_ID); taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID)); } else { - throw new ElasticsearchStatusException("Inference ID must be provided in the path", RestStatus.BAD_REQUEST); + inferenceEntityId = restRequest.param(TASK_TYPE_OR_INFERENCE_ID); + taskType = TaskType.ANY; } var content = restRequest.requiredContent(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java index e27fa8dcae518..23565d7370230 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java @@ -36,15 +36,19 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.XContentFormattedException; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Iterator; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Flow; import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.xpack.core.inference.results.XContentFormattedException.X_CONTENT_PARAM; + /** * A version of {@link org.elasticsearch.rest.action.RestChunkedToXContentListener} that reads from a {@link Flow.Publisher} and encodes * the response in Server-Sent Events. @@ -73,7 +77,7 @@ public ServerSentEventsRestActionListener(RestChannel channel, SetOnce threadPool) { this.channel = channel; - this.params = params; + this.params = new ToXContent.DelegatingMapParams(Map.of(X_CONTENT_PARAM, String.valueOf(channel.detailedErrorsEnabled())), params); this.threadPool = Objects.requireNonNull(threadPool); } @@ -151,6 +155,12 @@ public void onFailure(Exception e) { } private ChunkedToXContent errorChunk(Throwable t) { + // if we've already formatted it, just return that format + if (ExceptionsHelper.unwrapCause(t) instanceof XContentFormattedException xContentFormattedException) { + return xContentFormattedException; + } + + // else, try to parse the format and return something that the ES client knows how to interpret var status = ExceptionsHelper.status(t); Exception e; @@ -159,7 +169,8 @@ private ChunkedToXContent errorChunk(Throwable t) { } else { // if not exception, then error, and we should not let it escape. rethrow on another thread, and inform the user we're stopping. ExceptionsHelper.maybeDieOnAnotherThread(t); - e = new RuntimeException("Fatal error while streaming response", t); + e = new RuntimeException("Fatal error while streaming response. Please retry the request."); + logger.error(e.getMessage(), t); } return params -> Iterators.concat( ChunkedToXContentHelper.startObject(), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 1ddae3cc8df95..13d641101a1cf 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -42,7 +42,7 @@ import static org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings.ENABLED; import static org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings.MAX_NUMBER_OF_ALLOCATIONS; import static org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings.MIN_NUMBER_OF_ALLOCATIONS; -import static org.elasticsearch.xpack.inference.rest.Paths.UNIFIED_SUFFIX; +import static org.elasticsearch.xpack.inference.rest.Paths.STREAM_SUFFIX; import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY; public final class ServiceUtils { @@ -796,7 +796,7 @@ public static String useChatCompletionUrlMessage(Model model) { model.getTaskType(), model.getTaskType(), model.getInferenceEntityId(), - UNIFIED_SUFFIX + STREAM_SUFFIX ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index 0fd0c281d8bc6..589ca1e033f06 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -409,7 +409,7 @@ public static InferenceServiceConfiguration get() { HTTP_SCHEMA_NAME, new SettingsConfiguration.Builder(supportedTaskTypes).setDescription("") .setLabel("HTTP Schema") - .setRequired(true) + .setRequired(false) .setSensitive(false) .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index e13c668197a8f..493acd3c0cd1a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -413,6 +414,20 @@ public static InferenceServiceConfiguration get() { .build() ); + configurationMap.put( + DIMENSIONS, + new SettingsConfiguration.Builder(EnumSet.of(TaskType.TEXT_EMBEDDING)).setDescription( + "The number of dimensions the resulting embeddings should have. For more information refer to " + + "https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html." + ) + .setLabel("Dimensions") + .setRequired(false) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.INTEGER) + .build() + ); + configurationMap.putAll(AmazonBedrockSecretSettings.Configuration.get()); configurationMap.putAll( RateLimitSettings.toSettingsConfigurationWithDescription( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java index 88d5b54398d06..34a5c2b4cc1e9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java @@ -53,6 +53,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -441,6 +442,20 @@ public static InferenceServiceConfiguration get() { .build() ); + configurationMap.put( + DIMENSIONS, + new SettingsConfiguration.Builder(EnumSet.of(TaskType.TEXT_EMBEDDING)).setDescription( + "The number of dimensions the resulting embeddings should have. For more information refer to " + + "https://learn.microsoft.com/en-us/azure/ai-studio/reference/reference-model-inference-embeddings." + ) + .setLabel("Dimensions") + .setRequired(false) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.INTEGER) + .build() + ); + configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration(supportedTaskTypes)); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration(supportedTaskTypes)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index 5b622d68f2c25..9a77b63337978 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -50,6 +50,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -382,6 +383,20 @@ public static InferenceServiceConfiguration get() { .build() ); + configurationMap.put( + DIMENSIONS, + new SettingsConfiguration.Builder(EnumSet.of(TaskType.TEXT_EMBEDDING)).setDescription( + "The number of dimensions the resulting embeddings should have. For more information refer to " + + "https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-body-1." + ) + .setLabel("Dimensions") + .setRequired(false) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.INTEGER) + .build() + ); + configurationMap.putAll(AzureOpenAiSecretSettings.Configuration.get()); configurationMap.putAll( RateLimitSettings.toSettingsConfigurationWithDescription( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 60326a8a34ca3..b32cec54df414 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceConfiguration; @@ -25,6 +26,7 @@ import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; @@ -51,6 +53,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -333,15 +336,15 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { } /** - * Return the default similarity measure for the embedding type. - * Cohere embeddings are normalized to unit vectors therefor Dot - * Product similarity can be used and is the default for all Cohere - * models. + * Returns the default similarity measure for the embedding type. + * Cohere embeddings are expected to be normalized to unit vectors, but due to floating point precision issues, + * our check ({@link DenseVectorFieldMapper#isNotUnitVector(float)}) often fails. + * Therefore, we use cosine similarity to ensure compatibility. * - * @return The default similarity. + * @return The default similarity measure. */ static SimilarityMeasure defaultSimilarity() { - return SimilarityMeasure.DOT_PRODUCT; + return SimilarityMeasure.COSINE; } @Override @@ -363,6 +366,19 @@ public static InferenceServiceConfiguration get() { () -> { var configurationMap = new HashMap(); + configurationMap.put( + MODEL_ID, + new SettingsConfiguration.Builder(supportedTaskTypes).setDescription( + "The name of the model to use for the inference task." + ) + .setLabel("Model ID") + .setRequired(false) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.STRING) + .build() + ); + configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration(supportedTaskTypes)); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration(supportedTaskTypes)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/DefaultModelConfig.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/DefaultModelConfig.java new file mode 100644 index 0000000000000..dcdf5bce1fbb4 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/DefaultModelConfig.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.inference.MinimalServiceSettings; +import org.elasticsearch.inference.Model; + +public record DefaultModelConfig(Model model, MinimalServiceSettings settings) { + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java index f010c2f85a063..62c288d5eb8dc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.inference.services.elastic; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -49,25 +47,19 @@ import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.ServiceUtils; -import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorization; import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationHandler; +import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationRequestHandler; import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionModel; import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; import org.elasticsearch.xpack.inference.telemetry.TraceContext; -import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -85,7 +77,6 @@ public class ElasticInferenceService extends SenderService { public static final String NAME = "elastic"; public static final String ELASTIC_INFERENCE_SERVICE_IDENTIFIER = "Elastic Inference Service"; - private static final Logger logger = LogManager.getLogger(ElasticInferenceService.class); private static final EnumSet IMPLEMENTED_TASK_TYPES = EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION); private static final String SERVICE_NAME = "Elastic"; static final String DEFAULT_CHAT_COMPLETION_MODEL_ID_V1 = "rainbow-sprinkles"; @@ -96,32 +87,34 @@ public class ElasticInferenceService extends SenderService { */ private static final EnumSet SUPPORTED_INFERENCE_ACTION_TASK_TYPES = EnumSet.of(TaskType.SPARSE_EMBEDDING); + public static String defaultEndpointId(String modelId) { + return Strings.format(".%s-elastic", modelId); + } + private final ElasticInferenceServiceComponents elasticInferenceServiceComponents; - private Configuration configuration; - private final AtomicReference authRef = new AtomicReference<>(AuthorizedContent.empty()); - private final ModelRegistry modelRegistry; private final ElasticInferenceServiceAuthorizationHandler authorizationHandler; - private final CountDownLatch authorizationCompletedLatch = new CountDownLatch(1); - // model ids to model information, used for the default config methods to return the list of models and default - // configs - private final Map defaultModelsConfigs; public ElasticInferenceService( HttpRequestSender.Factory factory, ServiceComponents serviceComponents, - ElasticInferenceServiceComponents elasticInferenceServiceComponents, + ElasticInferenceServiceSettings elasticInferenceServiceSettings, ModelRegistry modelRegistry, - ElasticInferenceServiceAuthorizationHandler authorizationHandler + ElasticInferenceServiceAuthorizationRequestHandler authorizationRequestHandler ) { super(factory, serviceComponents); - this.elasticInferenceServiceComponents = Objects.requireNonNull(elasticInferenceServiceComponents); - this.modelRegistry = Objects.requireNonNull(modelRegistry); - this.authorizationHandler = Objects.requireNonNull(authorizationHandler); - - configuration = new Configuration(authRef.get().taskTypesAndModels.getAuthorizedTaskTypes()); - defaultModelsConfigs = initDefaultEndpoints(elasticInferenceServiceComponents); - - getAuthorization(); + this.elasticInferenceServiceComponents = new ElasticInferenceServiceComponents( + elasticInferenceServiceSettings.getElasticInferenceServiceUrl() + ); + authorizationHandler = new ElasticInferenceServiceAuthorizationHandler( + serviceComponents, + modelRegistry, + authorizationRequestHandler, + initDefaultEndpoints(elasticInferenceServiceComponents), + IMPLEMENTED_TASK_TYPES, + this, + getSender(), + elasticInferenceServiceSettings + ); } private static Map initDefaultEndpoints( @@ -144,126 +137,35 @@ private static Map initDefaultEndpoints( ); } - private record DefaultModelConfig(Model model, MinimalServiceSettings settings) {} - - private record AuthorizedContent( - ElasticInferenceServiceAuthorization taskTypesAndModels, - List configIds, - List defaultModelConfigs - ) { - static AuthorizedContent empty() { - return new AuthorizedContent(ElasticInferenceServiceAuthorization.newDisabledService(), List.of(), List.of()); - } - } - - private void getAuthorization() { - try { - ActionListener listener = ActionListener.wrap(result -> { - setAuthorizedContent(result); - authorizationCompletedLatch.countDown(); - }, e -> { - // we don't need to do anything if there was a failure, everything is disabled by default - authorizationCompletedLatch.countDown(); - }); - - authorizationHandler.getAuthorization(listener, getSender()); - } catch (Exception e) { - // we don't need to do anything if there was a failure, everything is disabled by default - authorizationCompletedLatch.countDown(); - } - } - - private synchronized void setAuthorizedContent(ElasticInferenceServiceAuthorization auth) { - var authorizedTaskTypesAndModels = auth.newLimitedToTaskTypes(EnumSet.copyOf(IMPLEMENTED_TASK_TYPES)); - - // recalculate which default config ids and models are authorized now - var authorizedDefaultConfigIds = getAuthorizedDefaultConfigIds(auth); - var authorizedDefaultModelObjects = getAuthorizedDefaultModelsObjects(auth); - authRef.set(new AuthorizedContent(authorizedTaskTypesAndModels, authorizedDefaultConfigIds, authorizedDefaultModelObjects)); - - configuration = new Configuration(authRef.get().taskTypesAndModels.getAuthorizedTaskTypes()); - - defaultConfigIds().forEach(modelRegistry::addDefaultIds); - } - - private List getAuthorizedDefaultConfigIds(ElasticInferenceServiceAuthorization auth) { - var authorizedDefaultModelIds = getAuthorizedDefaultModelIds(auth); - - var authorizedConfigIds = new ArrayList(); - for (var id : authorizedDefaultModelIds) { - var modelConfig = defaultModelsConfigs.get(id); - if (modelConfig != null) { - if (auth.getAuthorizedTaskTypes().contains(modelConfig.model.getTaskType()) == false) { - logger.warn( - Strings.format( - "The authorization response included the default model: %s, " - + "but did not authorize the assumed task type of the model: %s. Enabling model.", - id, - modelConfig.model.getTaskType() - ) - ); - } - authorizedConfigIds.add(new DefaultConfigId(modelConfig.model.getInferenceEntityId(), modelConfig.settings(), this)); - } - } - - return authorizedConfigIds; - } - - private Set getAuthorizedDefaultModelIds(ElasticInferenceServiceAuthorization auth) { - var authorizedModels = auth.getAuthorizedModelIds(); - var authorizedDefaultModelIds = new HashSet<>(defaultModelsConfigs.keySet()); - authorizedDefaultModelIds.retainAll(authorizedModels); - - return authorizedDefaultModelIds; - } - - private List getAuthorizedDefaultModelsObjects(ElasticInferenceServiceAuthorization auth) { - var authorizedDefaultModelIds = getAuthorizedDefaultModelIds(auth); - - var authorizedModels = new ArrayList(); - for (var id : authorizedDefaultModelIds) { - var modelConfig = defaultModelsConfigs.get(id); - if (modelConfig != null) { - authorizedModels.add(modelConfig); - } - } - - return authorizedModels; + @Override + public void onNodeStarted() { + authorizationHandler.init(); } - // Default for testing - void waitForAuthorizationToComplete(TimeValue waitTime) { - try { - if (authorizationCompletedLatch.await(waitTime.getSeconds(), TimeUnit.SECONDS) == false) { - throw new IllegalStateException("The wait time has expired for authorization to complete."); - } - } catch (InterruptedException e) { - throw new IllegalStateException("Waiting for authorization to complete was interrupted"); - } + /** + * Only use this in tests. + * + * Waits the specified amount of time for the authorization call to complete. This is mainly to make testing easier. + * @param waitTime the max time to wait + * @throws IllegalStateException if the wait time is exceeded or the call receives an {@link InterruptedException} + */ + public void waitForFirstAuthorizationToComplete(TimeValue waitTime) { + authorizationHandler.waitForAuthorizationToComplete(waitTime); } @Override - public synchronized Set supportedStreamingTasks() { - var authorizedStreamingTaskTypes = EnumSet.of(TaskType.CHAT_COMPLETION); - authorizedStreamingTaskTypes.retainAll(authRef.get().taskTypesAndModels.getAuthorizedTaskTypes()); - - if (authorizedStreamingTaskTypes.isEmpty() == false) { - authorizedStreamingTaskTypes.add(TaskType.ANY); - } - - return authorizedStreamingTaskTypes; + public Set supportedStreamingTasks() { + return authorizationHandler.supportedStreamingTasks(); } @Override - public synchronized List defaultConfigIds() { - return authRef.get().configIds; + public List defaultConfigIds() { + return authorizationHandler.defaultConfigIds(); } @Override - public synchronized void defaultConfigs(ActionListener> defaultsListener) { - var models = authRef.get().defaultModelConfigs.stream().map(config -> config.model).toList(); - defaultsListener.onResponse(models); + public void defaultConfigs(ActionListener> defaultsListener) { + authorizationHandler.defaultConfigs(defaultsListener); } @Override @@ -286,7 +188,6 @@ protected void doUnifiedCompletionInfer( var completionModel = (ElasticInferenceServiceCompletionModel) model; var overriddenModel = ElasticInferenceServiceCompletionModel.of(completionModel, inputs.getRequest()); var errorMessage = constructFailedToSendRequestMessage( - overriddenModel.uri(), String.format(Locale.ROOT, "%s completions", ELASTIC_INFERENCE_SERVICE_IDENTIFIER) ); @@ -316,6 +217,7 @@ protected void doInfer( responseString = responseString + " " + useChatCompletionUrlMessage(model); } listener.onFailure(new ElasticsearchStatusException(responseString, RestStatus.BAD_REQUEST)); + return; } if (model instanceof ElasticInferenceServiceExecutableActionModel == false) { @@ -390,18 +292,18 @@ public void parseRequestConfig( } @Override - public synchronized InferenceServiceConfiguration getConfiguration() { - return configuration.get(); + public InferenceServiceConfiguration getConfiguration() { + return authorizationHandler.getConfiguration(); } @Override - public synchronized EnumSet supportedTaskTypes() { - return authRef.get().taskTypesAndModels.getAuthorizedTaskTypes(); + public EnumSet supportedTaskTypes() { + return authorizationHandler.supportedTaskTypes(); } @Override - public synchronized boolean hideFromConfigurationApi() { - return authRef.get().taskTypesAndModels.isAuthorized() == false; + public boolean hideFromConfigurationApi() { + return authorizationHandler.hideFromConfigurationApi(); } private static ElasticInferenceServiceModel createModel( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java index 837581667882d..83fd957f9005d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java @@ -9,4 +9,13 @@ import org.elasticsearch.core.Nullable; -public record ElasticInferenceServiceComponents(@Nullable String elasticInferenceServiceUrl) {} +/** + * @param elasticInferenceServiceUrl the upstream Elastic Inference Server's URL + */ +public record ElasticInferenceServiceComponents(@Nullable String elasticInferenceServiceUrl) { + public static final ElasticInferenceServiceComponents EMPTY_INSTANCE = ElasticInferenceServiceComponents.of(null); + + public static ElasticInferenceServiceComponents of(String elasticInferenceServiceUrl) { + return new ElasticInferenceServiceComponents(elasticInferenceServiceUrl); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java index 98d55fd799598..fe6ebb6cfb625 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java @@ -7,9 +7,11 @@ package org.elasticsearch.xpack.inference.services.elastic; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import java.util.ArrayList; @@ -31,15 +33,31 @@ public class ElasticInferenceServiceSettings { Setting.Property.NodeScope ); - @Deprecated - private final String eisGatewayUrl; + /** + * This setting is for testing only. It controls whether authorization is only performed once at bootup. If set to true, an + * authorization request will be made repeatedly on an interval. + */ + static final Setting PERIODIC_AUTHORIZATION_ENABLED = Setting.boolSetting( + "xpack.inference.elastic.periodic_authorization_enabled", + true, + Setting.Property.NodeScope + ); - private final String elasticInferenceServiceUrl; + private static final TimeValue DEFAULT_AUTH_REQUEST_INTERVAL = TimeValue.timeValueMinutes(10); + static final Setting AUTHORIZATION_REQUEST_INTERVAL = Setting.timeSetting( + "xpack.inference.elastic.authorization_request_interval", + DEFAULT_AUTH_REQUEST_INTERVAL, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); - public ElasticInferenceServiceSettings(Settings settings) { - eisGatewayUrl = EIS_GATEWAY_URL.get(settings); - elasticInferenceServiceUrl = ELASTIC_INFERENCE_SERVICE_URL.get(settings); - } + private static final TimeValue DEFAULT_AUTH_REQUEST_JITTER = TimeValue.timeValueMinutes(5); + static final Setting MAX_AUTHORIZATION_REQUEST_JITTER = Setting.timeSetting( + "xpack.inference.elastic.max_authorization_request_jitter", + DEFAULT_AUTH_REQUEST_JITTER, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); public static final SSLConfigurationSettings ELASTIC_INFERENCE_SERVICE_SSL_CONFIGURATION_SETTINGS = SSLConfigurationSettings.withPrefix( ELASTIC_INFERENCE_SERVICE_SSL_CONFIGURATION_PREFIX, @@ -52,13 +70,60 @@ public ElasticInferenceServiceSettings(Settings settings) { Setting.Property.NodeScope ); + @Deprecated + private final String eisGatewayUrl; + + private final String elasticInferenceServiceUrl; + private final boolean periodicAuthorizationEnabled; + private volatile TimeValue authRequestInterval; + private volatile TimeValue maxAuthorizationRequestJitter; + + public ElasticInferenceServiceSettings(Settings settings) { + eisGatewayUrl = EIS_GATEWAY_URL.get(settings); + elasticInferenceServiceUrl = ELASTIC_INFERENCE_SERVICE_URL.get(settings); + periodicAuthorizationEnabled = PERIODIC_AUTHORIZATION_ENABLED.get(settings); + authRequestInterval = AUTHORIZATION_REQUEST_INTERVAL.get(settings); + maxAuthorizationRequestJitter = MAX_AUTHORIZATION_REQUEST_JITTER.get(settings); + } + + /** + * This must be called after the object is constructed to avoid leaking the this reference before the constructor + * finishes. + * + * Handles initializing the settings changes listener. + */ + public final void init(ClusterService clusterService) { + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(AUTHORIZATION_REQUEST_INTERVAL, this::setAuthorizationRequestInterval); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(MAX_AUTHORIZATION_REQUEST_JITTER, this::setMaxAuthorizationRequestJitter); + } + + private void setAuthorizationRequestInterval(TimeValue interval) { + authRequestInterval = interval; + } + + private void setMaxAuthorizationRequestJitter(TimeValue jitter) { + maxAuthorizationRequestJitter = jitter; + } + + public TimeValue getAuthRequestInterval() { + return authRequestInterval; + } + + public TimeValue getMaxAuthorizationRequestJitter() { + return maxAuthorizationRequestJitter; + } + public static List> getSettingsDefinitions() { ArrayList> settings = new ArrayList<>(); settings.add(EIS_GATEWAY_URL); settings.add(ELASTIC_INFERENCE_SERVICE_URL); settings.add(ELASTIC_INFERENCE_SERVICE_SSL_ENABLED); settings.addAll(ELASTIC_INFERENCE_SERVICE_SSL_CONFIGURATION_SETTINGS.getEnabledSettings()); - + settings.add(PERIODIC_AUTHORIZATION_ENABLED); + settings.add(AUTHORIZATION_REQUEST_INTERVAL); + settings.add(MAX_AUTHORIZATION_REQUEST_JITTER); return settings; } @@ -66,4 +131,7 @@ public String getElasticInferenceServiceUrl() { return Strings.isEmpty(elasticInferenceServiceUrl) ? eisGatewayUrl : elasticInferenceServiceUrl; } + public boolean isPeriodicAuthorizationEnabled() { + return periodicAuthorizationEnabled; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java index ac6a389914a10..fd38d63f7f74e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java @@ -58,7 +58,7 @@ public ElasticInferenceServiceSparseEmbeddingsModel( this.uri = createUri(); } - ElasticInferenceServiceSparseEmbeddingsModel( + public ElasticInferenceServiceSparseEmbeddingsModel( String inferenceEntityId, TaskType taskType, String service, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java index f78b5357caeb3..a3b80cd216067 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandler.java @@ -9,129 +9,339 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchWrapperException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.elastic.ElasticInferenceServiceResponseHandler; -import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceConfiguration; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.external.request.elastic.ElasticInferenceServiceAuthorizationRequest; -import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceAuthorizationResponseEntity; -import org.elasticsearch.xpack.inference.telemetry.TraceContext; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.elastic.DefaultModelConfig; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettings; -import java.util.Locale; +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; -import static org.elasticsearch.xpack.core.inference.action.InferenceAction.Request.DEFAULT_TIMEOUT; -import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService.ELASTIC_INFERENCE_SERVICE_IDENTIFIER; +import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; -/** - * Handles retrieving the authorization information from Elastic Inference Service. - */ -public class ElasticInferenceServiceAuthorizationHandler { +public class ElasticInferenceServiceAuthorizationHandler implements Closeable { + private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceAuthorizationHandler.class); - private static final String FAILED_TO_RETRIEVE_MESSAGE = - "Failed to retrieve the authorization information from the Elastic Inference Service."; - private static final ResponseHandler AUTH_RESPONSE_HANDLER = createAuthResponseHandler(); + private record AuthorizedContent( + ElasticInferenceServiceAuthorizationModel taskTypesAndModels, + List configIds, + List defaultModelConfigs + ) { + static AuthorizedContent empty() { + return new AuthorizedContent(ElasticInferenceServiceAuthorizationModel.newDisabledService(), List.of(), List.of()); + } + } - private static ResponseHandler createAuthResponseHandler() { - return new ElasticInferenceServiceResponseHandler( - String.format(Locale.ROOT, "%s sparse embeddings", ELASTIC_INFERENCE_SERVICE_IDENTIFIER), - ElasticInferenceServiceAuthorizationResponseEntity::fromResponse + private final ServiceComponents serviceComponents; + private final AtomicReference authorizedContent = new AtomicReference<>(AuthorizedContent.empty()); + private final ModelRegistry modelRegistry; + private final ElasticInferenceServiceAuthorizationRequestHandler authorizationHandler; + private final AtomicReference configuration; + private final Map defaultModelsConfigs; + private final CountDownLatch firstAuthorizationCompletedLatch = new CountDownLatch(1); + private final EnumSet implementedTaskTypes; + private final InferenceService inferenceService; + private final Sender sender; + private final Runnable callback; + private final AtomicReference lastAuthTask = new AtomicReference<>(null); + private final AtomicBoolean shutdown = new AtomicBoolean(false); + private final ElasticInferenceServiceSettings elasticInferenceServiceSettings; + + public ElasticInferenceServiceAuthorizationHandler( + ServiceComponents serviceComponents, + ModelRegistry modelRegistry, + ElasticInferenceServiceAuthorizationRequestHandler authorizationRequestHandler, + Map defaultModelsConfigs, + EnumSet implementedTaskTypes, + InferenceService inferenceService, + Sender sender, + ElasticInferenceServiceSettings elasticInferenceServiceSettings + ) { + this( + serviceComponents, + modelRegistry, + authorizationRequestHandler, + defaultModelsConfigs, + implementedTaskTypes, + Objects.requireNonNull(inferenceService), + sender, + elasticInferenceServiceSettings, + null ); } - private final String baseUrl; - private final ThreadPool threadPool; - private final Logger logger; - private final CountDownLatch requestCompleteLatch = new CountDownLatch(1); + // default for testing + ElasticInferenceServiceAuthorizationHandler( + ServiceComponents serviceComponents, + ModelRegistry modelRegistry, + ElasticInferenceServiceAuthorizationRequestHandler authorizationRequestHandler, + Map defaultModelsConfigs, + EnumSet implementedTaskTypes, + InferenceService inferenceService, + Sender sender, + ElasticInferenceServiceSettings elasticInferenceServiceSettings, + // this is a hack to facilitate testing + Runnable callback + ) { + this.serviceComponents = Objects.requireNonNull(serviceComponents); + this.modelRegistry = Objects.requireNonNull(modelRegistry); + this.authorizationHandler = Objects.requireNonNull(authorizationRequestHandler); + this.defaultModelsConfigs = Objects.requireNonNull(defaultModelsConfigs); + this.implementedTaskTypes = Objects.requireNonNull(implementedTaskTypes); + // allow the service to be null for testing + this.inferenceService = inferenceService; + this.sender = Objects.requireNonNull(sender); + this.elasticInferenceServiceSettings = Objects.requireNonNull(elasticInferenceServiceSettings); - public ElasticInferenceServiceAuthorizationHandler(@Nullable String baseUrl, ThreadPool threadPool) { - this.baseUrl = baseUrl; - this.threadPool = Objects.requireNonNull(threadPool); - logger = LogManager.getLogger(ElasticInferenceServiceAuthorizationHandler.class); + configuration = new AtomicReference<>( + new ElasticInferenceService.Configuration(authorizedContent.get().taskTypesAndModels.getAuthorizedTaskTypes()) + ); + this.callback = callback; } - // only use for testing - ElasticInferenceServiceAuthorizationHandler(@Nullable String baseUrl, ThreadPool threadPool, Logger logger) { - this.baseUrl = baseUrl; - this.threadPool = Objects.requireNonNull(threadPool); - this.logger = Objects.requireNonNull(logger); + public void init() { + logger.debug("Initializing authorization logic"); + serviceComponents.threadPool().executor(UTILITY_THREAD_POOL_NAME).execute(this::scheduleAndSendAuthorizationRequest); } /** - * Retrieve the authorization information from Elastic Inference Service - * @param listener a listener to receive the response - * @param sender a {@link Sender} for making the request to the Elastic Inference Service + * Waits the specified amount of time for the first authorization call to complete. This is mainly to make testing easier. + * @param waitTime the max time to wait + * @throws IllegalStateException if the wait time is exceeded or the call receives an {@link InterruptedException} */ - public void getAuthorization(ActionListener listener, Sender sender) { + public void waitForAuthorizationToComplete(TimeValue waitTime) { try { - logger.debug("Retrieving authorization information from the Elastic Inference Service."); + if (firstAuthorizationCompletedLatch.await(waitTime.getSeconds(), TimeUnit.SECONDS) == false) { + throw new IllegalStateException("The wait time has expired for authorization to complete."); + } + } catch (InterruptedException e) { + throw new IllegalStateException("Waiting for authorization to complete was interrupted"); + } + } + + public synchronized Set supportedStreamingTasks() { + var authorizedStreamingTaskTypes = EnumSet.of(TaskType.CHAT_COMPLETION); + authorizedStreamingTaskTypes.retainAll(authorizedContent.get().taskTypesAndModels.getAuthorizedTaskTypes()); + + if (authorizedStreamingTaskTypes.isEmpty() == false) { + authorizedStreamingTaskTypes.add(TaskType.ANY); + } + + return authorizedStreamingTaskTypes; + } + + public synchronized List defaultConfigIds() { + return authorizedContent.get().configIds; + } - if (Strings.isNullOrEmpty(baseUrl)) { - logger.warn("The base URL for the authorization service is not valid, rejecting authorization."); - listener.onResponse(ElasticInferenceServiceAuthorization.newDisabledService()); + public synchronized void defaultConfigs(ActionListener> defaultsListener) { + var models = authorizedContent.get().defaultModelConfigs.stream().map(DefaultModelConfig::model).toList(); + defaultsListener.onResponse(models); + } + + public synchronized EnumSet supportedTaskTypes() { + return authorizedContent.get().taskTypesAndModels.getAuthorizedTaskTypes(); + } + + public synchronized boolean hideFromConfigurationApi() { + return authorizedContent.get().taskTypesAndModels.isAuthorized() == false; + } + + public synchronized InferenceServiceConfiguration getConfiguration() { + return configuration.get().get(); + } + + @Override + public void close() throws IOException { + shutdown.set(true); + if (lastAuthTask.get() != null) { + lastAuthTask.get().cancel(); + } + } + + private void scheduleAuthorizationRequest() { + try { + if (elasticInferenceServiceSettings.isPeriodicAuthorizationEnabled() == false) { return; } - // ensure that the sender is initialized - sender.start(); + // this call has to be on the individual thread otherwise we get an exception + var random = Randomness.get(); + var jitter = (long) (elasticInferenceServiceSettings.getMaxAuthorizationRequestJitter().millis() * random.nextDouble()); + var waitTime = TimeValue.timeValueMillis(elasticInferenceServiceSettings.getAuthRequestInterval().millis() + jitter); - ActionListener newListener = ActionListener.wrap(results -> { - if (results instanceof ElasticInferenceServiceAuthorizationResponseEntity authResponseEntity) { - listener.onResponse(ElasticInferenceServiceAuthorization.of(authResponseEntity)); - } else { - logger.warn( - Strings.format( - FAILED_TO_RETRIEVE_MESSAGE + " Received an invalid response type: %s", - results.getClass().getSimpleName() - ) - ); - listener.onResponse(ElasticInferenceServiceAuthorization.newDisabledService()); + logger.debug( + () -> Strings.format( + "Scheduling the next authorization call with request interval: %s ms, jitter: %d ms", + elasticInferenceServiceSettings.getAuthRequestInterval().millis(), + jitter + ) + ); + logger.debug(() -> Strings.format("Next authorization call in %d minutes", waitTime.getMinutes())); + + lastAuthTask.set( + serviceComponents.threadPool() + .schedule( + this::scheduleAndSendAuthorizationRequest, + waitTime, + serviceComponents.threadPool().executor(UTILITY_THREAD_POOL_NAME) + ) + ); + } catch (Exception e) { + logger.warn("Failed scheduling authorization request", e); + } + } + + private void scheduleAndSendAuthorizationRequest() { + if (shutdown.get()) { + return; + } + + scheduleAuthorizationRequest(); + sendAuthorizationRequest(); + } + + private void sendAuthorizationRequest() { + try { + ActionListener listener = ActionListener.wrap((model) -> { + setAuthorizedContent(model); + if (callback != null) { + callback.run(); } - requestCompleteLatch.countDown(); }, e -> { - Throwable exception = e; - if (e instanceof ElasticsearchWrapperException wrapperException) { - exception = wrapperException.getCause(); - } - - logger.warn(Strings.format(FAILED_TO_RETRIEVE_MESSAGE + " Encountered an exception: %s", exception)); - listener.onResponse(ElasticInferenceServiceAuthorization.newDisabledService()); - requestCompleteLatch.countDown(); + // we don't need to do anything if there was a failure, everything is disabled by default + firstAuthorizationCompletedLatch.countDown(); }); - var request = new ElasticInferenceServiceAuthorizationRequest(baseUrl, getCurrentTraceInfo()); - - sender.sendWithoutQueuing(logger, request, AUTH_RESPONSE_HANDLER, DEFAULT_TIMEOUT, newListener); + authorizationHandler.getAuthorization(listener, sender); } catch (Exception e) { - logger.warn(Strings.format("Retrieving the authorization information encountered an exception: %s", e)); - requestCompleteLatch.countDown(); + logger.warn("Failure while sending the request to retrieve authorization", e); + // we don't need to do anything if there was a failure, everything is disabled by default + firstAuthorizationCompletedLatch.countDown(); } } - private TraceContext getCurrentTraceInfo() { - var traceParent = threadPool.getThreadContext().getHeader(Task.TRACE_PARENT); - var traceState = threadPool.getThreadContext().getHeader(Task.TRACE_STATE); + private synchronized void setAuthorizedContent(ElasticInferenceServiceAuthorizationModel auth) { + logger.debug("Received authorization response"); + var authorizedTaskTypesAndModels = authorizedContent.get().taskTypesAndModels.merge(auth) + .newLimitedToTaskTypes(EnumSet.copyOf(implementedTaskTypes)); + + // recalculate which default config ids and models are authorized now + var authorizedDefaultModelIds = getAuthorizedDefaultModelIds(auth); + + var authorizedDefaultConfigIds = getAuthorizedDefaultConfigIds(authorizedDefaultModelIds, auth); + var authorizedDefaultModelObjects = getAuthorizedDefaultModelsObjects(authorizedDefaultModelIds); + authorizedContent.set( + new AuthorizedContent(authorizedTaskTypesAndModels, authorizedDefaultConfigIds, authorizedDefaultModelObjects) + ); + + configuration.set(new ElasticInferenceService.Configuration(authorizedContent.get().taskTypesAndModels.getAuthorizedTaskTypes())); - return new TraceContext(traceParent, traceState); + authorizedContent.get().configIds().forEach(modelRegistry::putDefaultIdIfAbsent); + handleRevokedDefaultConfigs(authorizedDefaultModelIds); } - // Default because should only be used for testing - void waitForAuthRequestCompletion(TimeValue timeValue) throws IllegalStateException { - try { - if (requestCompleteLatch.await(timeValue.getMillis(), TimeUnit.MILLISECONDS) == false) { - throw new IllegalStateException("The wait time has expired for authorization to complete."); + private Set getAuthorizedDefaultModelIds(ElasticInferenceServiceAuthorizationModel auth) { + var authorizedModels = auth.getAuthorizedModelIds(); + var authorizedDefaultModelIds = new TreeSet<>(defaultModelsConfigs.keySet()); + authorizedDefaultModelIds.retainAll(authorizedModels); + + return authorizedDefaultModelIds; + } + + private List getAuthorizedDefaultConfigIds( + Set authorizedDefaultModelIds, + ElasticInferenceServiceAuthorizationModel auth + ) { + var authorizedConfigIds = new ArrayList(); + for (var id : authorizedDefaultModelIds) { + var modelConfig = defaultModelsConfigs.get(id); + if (modelConfig != null) { + if (auth.getAuthorizedTaskTypes().contains(modelConfig.model().getTaskType()) == false) { + logger.warn( + org.elasticsearch.common.Strings.format( + "The authorization response included the default model: %s, " + + "but did not authorize the assumed task type of the model: %s. Enabling model.", + id, + modelConfig.model().getTaskType() + ) + ); + } + authorizedConfigIds.add( + new InferenceService.DefaultConfigId( + modelConfig.model().getInferenceEntityId(), + modelConfig.settings(), + inferenceService + ) + ); + } + } + + authorizedConfigIds.sort(Comparator.comparing(InferenceService.DefaultConfigId::inferenceId)); + return authorizedConfigIds; + } + + private List getAuthorizedDefaultModelsObjects(Set authorizedDefaultModelIds) { + var authorizedModels = new ArrayList(); + for (var id : authorizedDefaultModelIds) { + var modelConfig = defaultModelsConfigs.get(id); + if (modelConfig != null) { + authorizedModels.add(modelConfig); } - } catch (InterruptedException e) { - throw new IllegalStateException("Waiting for authorization to complete was interrupted"); } + + authorizedModels.sort(Comparator.comparing(modelConfig -> modelConfig.model().getInferenceEntityId())); + return authorizedModels; + } + + private void handleRevokedDefaultConfigs(Set authorizedDefaultModelIds) { + // if a model was initially returned in the authorization response but is absent, then we'll assume authorization was revoked + var unauthorizedDefaultModelIds = new HashSet<>(defaultModelsConfigs.keySet()); + unauthorizedDefaultModelIds.removeAll(authorizedDefaultModelIds); + + // get all the default inference endpoint ids for the unauthorized model ids + var unauthorizedDefaultInferenceEndpointIds = unauthorizedDefaultModelIds.stream() + .map(defaultModelsConfigs::get) // get all the model configs + .filter(Objects::nonNull) // limit to only non-null + .map(modelConfig -> modelConfig.model().getInferenceEntityId()) // get the inference ids + .collect(Collectors.toSet()); + + var deleteInferenceEndpointsListener = ActionListener.wrap(result -> { + logger.debug(Strings.format("Successfully revoked access to default inference endpoint IDs: %s", unauthorizedDefaultModelIds)); + firstAuthorizationCompletedLatch.countDown(); + }, e -> { + logger.warn( + Strings.format("Failed to revoke access to default inference endpoint IDs: %s, error: %s", unauthorizedDefaultModelIds, e) + ); + firstAuthorizationCompletedLatch.countDown(); + }); + + logger.debug("Synchronizing default inference endpoints"); + modelRegistry.removeDefaultConfigs(unauthorizedDefaultInferenceEndpointIds, deleteInferenceEndpointsListener); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorization.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationModel.java similarity index 63% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorization.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationModel.java index 76721bb6dcd7b..6ff3cb950151e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorization.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationModel.java @@ -19,21 +19,21 @@ import java.util.stream.Collectors; /** - * This is a helper class for managing the response from {@link ElasticInferenceServiceAuthorizationHandler}. + * Transforms the response from {@link ElasticInferenceServiceAuthorizationRequestHandler} into a format for consumption by the service. */ -public class ElasticInferenceServiceAuthorization { +public class ElasticInferenceServiceAuthorizationModel { private final Map> taskTypeToModels; private final EnumSet authorizedTaskTypes; private final Set authorizedModelIds; /** - * Converts an authorization response from Elastic Inference Service into the {@link ElasticInferenceServiceAuthorization} format. + * Converts an authorization response from Elastic Inference Service into the {@link ElasticInferenceServiceAuthorizationModel} format. * * @param responseEntity the {@link ElasticInferenceServiceAuthorizationResponseEntity} response from the upstream gateway. - * @return a new {@link ElasticInferenceServiceAuthorization} + * @return a new {@link ElasticInferenceServiceAuthorizationModel} */ - public static ElasticInferenceServiceAuthorization of(ElasticInferenceServiceAuthorizationResponseEntity responseEntity) { + public static ElasticInferenceServiceAuthorizationModel of(ElasticInferenceServiceAuthorizationResponseEntity responseEntity) { var taskTypeToModelsMap = new HashMap>(); var enabledTaskTypesSet = EnumSet.noneOf(TaskType.class); var enabledModelsSet = new HashSet(); @@ -54,17 +54,17 @@ public static ElasticInferenceServiceAuthorization of(ElasticInferenceServiceAut } } - return new ElasticInferenceServiceAuthorization(taskTypeToModelsMap, enabledModelsSet, enabledTaskTypesSet); + return new ElasticInferenceServiceAuthorizationModel(taskTypeToModelsMap, enabledModelsSet, enabledTaskTypesSet); } /** * Returns an object indicating that the cluster has no access to Elastic Inference Service. */ - public static ElasticInferenceServiceAuthorization newDisabledService() { - return new ElasticInferenceServiceAuthorization(Map.of(), Set.of(), EnumSet.noneOf(TaskType.class)); + public static ElasticInferenceServiceAuthorizationModel newDisabledService() { + return new ElasticInferenceServiceAuthorizationModel(Map.of(), Set.of(), EnumSet.noneOf(TaskType.class)); } - private ElasticInferenceServiceAuthorization( + private ElasticInferenceServiceAuthorizationModel( Map> taskTypeToModels, Set authorizedModelIds, EnumSet authorizedTaskTypes @@ -91,13 +91,13 @@ public EnumSet getAuthorizedTaskTypes() { } /** - * Returns a new {@link ElasticInferenceServiceAuthorization} object retaining only the specified task types + * Returns a new {@link ElasticInferenceServiceAuthorizationModel} object retaining only the specified task types * and applicable models that leverage those task types. Any task types not specified in the passed in set will be * excluded from the returned object. This is essentially an intersection. * @param taskTypes the task types to retain in the newly created object * @return a new object containing models and task types limited to the specified set. */ - public ElasticInferenceServiceAuthorization newLimitedToTaskTypes(EnumSet taskTypes) { + public ElasticInferenceServiceAuthorizationModel newLimitedToTaskTypes(EnumSet taskTypes) { var newTaskTypeToModels = new HashMap>(); var taskTypesThatHaveModels = EnumSet.noneOf(TaskType.class); @@ -110,15 +110,48 @@ public ElasticInferenceServiceAuthorization newLimitedToTaskTypes(EnumSet newEnabledModels = newTaskTypeToModels.values().stream().flatMap(Set::stream).collect(Collectors.toSet()); + return new ElasticInferenceServiceAuthorizationModel( + newTaskTypeToModels, + enabledModels(newTaskTypeToModels), + taskTypesThatHaveModels + ); + } + + private static Set enabledModels(Map> taskTypeToModels) { + return taskTypeToModels.values().stream().flatMap(Set::stream).collect(Collectors.toSet()); + } + + /** + * Returns a new {@link ElasticInferenceServiceAuthorizationModel} that combines the current model and the passed in one. + * @param other model to merge into this one + * @return a new model + */ + public ElasticInferenceServiceAuthorizationModel merge(ElasticInferenceServiceAuthorizationModel other) { + Map> newTaskTypeToModels = taskTypeToModels.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new HashSet<>(e.getValue()))); + + for (var entry : other.taskTypeToModels.entrySet()) { + newTaskTypeToModels.merge(entry.getKey(), new HashSet<>(entry.getValue()), (existingModelIds, newModelIds) -> { + existingModelIds.addAll(newModelIds); + return existingModelIds; + }); + } + + var newAuthorizedTaskTypes = authorizedTaskTypes.isEmpty() ? EnumSet.noneOf(TaskType.class) : EnumSet.copyOf(authorizedTaskTypes); + newAuthorizedTaskTypes.addAll(other.authorizedTaskTypes); - return new ElasticInferenceServiceAuthorization(newTaskTypeToModels, newEnabledModels, taskTypesThatHaveModels); + return new ElasticInferenceServiceAuthorizationModel( + newTaskTypeToModels, + enabledModels(newTaskTypeToModels), + newAuthorizedTaskTypes + ); } @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; - ElasticInferenceServiceAuthorization that = (ElasticInferenceServiceAuthorization) o; + ElasticInferenceServiceAuthorizationModel that = (ElasticInferenceServiceAuthorizationModel) o; return Objects.equals(taskTypeToModels, that.taskTypeToModels) && Objects.equals(authorizedTaskTypes, that.authorizedTaskTypes) && Objects.equals(authorizedModelIds, that.authorizedModelIds); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationRequestHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationRequestHandler.java new file mode 100644 index 0000000000000..77381fef98128 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationRequestHandler.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic.authorization; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchWrapperException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.elastic.ElasticInferenceServiceResponseHandler; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.request.elastic.ElasticInferenceServiceAuthorizationRequest; +import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceAuthorizationResponseEntity; +import org.elasticsearch.xpack.inference.telemetry.TraceContext; + +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService.ELASTIC_INFERENCE_SERVICE_IDENTIFIER; + +/** + * Handles retrieving the authorization information from Elastic Inference Service. + */ +public class ElasticInferenceServiceAuthorizationRequestHandler { + + private static final String FAILED_TO_RETRIEVE_MESSAGE = + "Failed to retrieve the authorization information from the Elastic Inference Service."; + private static final TimeValue DEFAULT_AUTH_TIMEOUT = TimeValue.timeValueMinutes(1); + private static final ResponseHandler AUTH_RESPONSE_HANDLER = createAuthResponseHandler(); + + private static ResponseHandler createAuthResponseHandler() { + return new ElasticInferenceServiceResponseHandler( + String.format(Locale.ROOT, "%s sparse embeddings", ELASTIC_INFERENCE_SERVICE_IDENTIFIER), + ElasticInferenceServiceAuthorizationResponseEntity::fromResponse + ); + } + + private final String baseUrl; + private final ThreadPool threadPool; + private final Logger logger; + private final CountDownLatch requestCompleteLatch = new CountDownLatch(1); + + public ElasticInferenceServiceAuthorizationRequestHandler(@Nullable String baseUrl, ThreadPool threadPool) { + this.baseUrl = baseUrl; + this.threadPool = Objects.requireNonNull(threadPool); + logger = LogManager.getLogger(ElasticInferenceServiceAuthorizationRequestHandler.class); + } + + // only use for testing + ElasticInferenceServiceAuthorizationRequestHandler(@Nullable String baseUrl, ThreadPool threadPool, Logger logger) { + this.baseUrl = baseUrl; + this.threadPool = Objects.requireNonNull(threadPool); + this.logger = Objects.requireNonNull(logger); + } + + /** + * Retrieve the authorization information from Elastic Inference Service + * @param listener a listener to receive the response + * @param sender a {@link Sender} for making the request to the Elastic Inference Service + */ + public void getAuthorization(ActionListener listener, Sender sender) { + try { + logger.debug("Retrieving authorization information from the Elastic Inference Service."); + + if (Strings.isNullOrEmpty(baseUrl)) { + logger.debug("The base URL for the authorization service is not valid, rejecting authorization."); + listener.onResponse(ElasticInferenceServiceAuthorizationModel.newDisabledService()); + return; + } + + // ensure that the sender is initialized + sender.start(); + + ActionListener newListener = ActionListener.wrap(results -> { + if (results instanceof ElasticInferenceServiceAuthorizationResponseEntity authResponseEntity) { + listener.onResponse(ElasticInferenceServiceAuthorizationModel.of(authResponseEntity)); + } else { + logger.warn( + Strings.format( + FAILED_TO_RETRIEVE_MESSAGE + " Received an invalid response type: %s", + results.getClass().getSimpleName() + ) + ); + listener.onResponse(ElasticInferenceServiceAuthorizationModel.newDisabledService()); + } + requestCompleteLatch.countDown(); + }, e -> { + Throwable exception = e; + if (e instanceof ElasticsearchWrapperException wrapperException) { + exception = wrapperException.getCause(); + } + + logger.warn(Strings.format(FAILED_TO_RETRIEVE_MESSAGE + " Encountered an exception: %s", exception)); + listener.onResponse(ElasticInferenceServiceAuthorizationModel.newDisabledService()); + requestCompleteLatch.countDown(); + }); + + var request = new ElasticInferenceServiceAuthorizationRequest(baseUrl, getCurrentTraceInfo()); + + sender.sendWithoutQueuing(logger, request, AUTH_RESPONSE_HANDLER, DEFAULT_AUTH_TIMEOUT, newListener); + } catch (Exception e) { + logger.warn(Strings.format("Retrieving the authorization information encountered an exception: %s", e)); + requestCompleteLatch.countDown(); + } + } + + private TraceContext getCurrentTraceInfo() { + var traceParent = threadPool.getThreadContext().getHeader(Task.TRACE_PARENT); + var traceState = threadPool.getThreadContext().getHeader(Task.TRACE_STATE); + + return new TraceContext(traceParent, traceState); + } + + // Default because should only be used for testing + void waitForAuthRequestCompletion(TimeValue timeValue) throws IllegalStateException { + try { + if (requestCompleteLatch.await(timeValue.getMillis(), TimeUnit.MILLISECONDS) == false) { + throw new IllegalStateException("The wait time has expired for authorization to complete."); + } + } catch (InterruptedException e) { + throw new IllegalStateException("Waiting for authorization to complete was interrupted"); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java index 3cc7e0c6c2b53..4591418419ded 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import java.io.IOException; @@ -20,15 +19,6 @@ public CustomElandInternalServiceSettings(ElasticsearchInternalServiceSettings o super(other); } - public CustomElandInternalServiceSettings( - Integer numAllocations, - int numThreads, - String modelId, - AdaptiveAllocationsSettings adaptiveAllocationsSettings - ) { - super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); - } - public CustomElandInternalServiceSettings(StreamInput in) throws IOException { super(in); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java index 133be5e2b7623..6395cd36457c2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.SimilarityMeasure; @@ -105,33 +106,17 @@ private static CommonFields commonFieldsFromMap(Map map, Validat private final SimilarityMeasure similarityMeasure; private final DenseVectorFieldMapper.ElementType elementType; - public CustomElandInternalTextEmbeddingServiceSettings( - int numAllocations, - int numThreads, - String modelId, - AdaptiveAllocationsSettings adaptiveAllocationsSettings - ) { - this( - numAllocations, - numThreads, - modelId, - adaptiveAllocationsSettings, - null, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ); - } - - public CustomElandInternalTextEmbeddingServiceSettings( - int numAllocations, + CustomElandInternalTextEmbeddingServiceSettings( + Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings, + @Nullable String deploymentId, Integer dimensions, SimilarityMeasure similarityMeasure, DenseVectorFieldMapper.ElementType elementType ) { - super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); + super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings, deploymentId); this.dimensions = dimensions; this.similarityMeasure = Objects.requireNonNull(similarityMeasure); this.elementType = Objects.requireNonNull(elementType); @@ -159,7 +144,8 @@ private CustomElandInternalTextEmbeddingServiceSettings(CommonFields commonField commonFields.internalServiceSettings.getNumAllocations(), commonFields.internalServiceSettings.getNumThreads(), commonFields.internalServiceSettings.modelId(), - commonFields.internalServiceSettings.getAdaptiveAllocationsSettings() + commonFields.internalServiceSettings.getAdaptiveAllocationsSettings(), + commonFields.internalServiceSettings.getDeploymentId() ); this.dimensions = dimensions; similarityMeasure = commonFields.similarityMeasure; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerServiceSettings.java index 316dc092e03c7..2b7904e615682 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerServiceSettings.java @@ -14,21 +14,27 @@ import java.io.IOException; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.RERANKER_ID; + public class ElasticRerankerServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "elastic_reranker_service_settings"; + public static ElasticRerankerServiceSettings defaultEndpointSettings() { + return new ElasticRerankerServiceSettings(null, 1, RERANKER_ID, new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32)); + } + public ElasticRerankerServiceSettings(ElasticsearchInternalServiceSettings other) { super(other); } - public ElasticRerankerServiceSettings( + private ElasticRerankerServiceSettings( Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings ) { - super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); + super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings, null); } public ElasticRerankerServiceSettings(StreamInput in) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index ac4dbe68a8f83..f4d361ab319e8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -15,8 +15,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; @@ -33,13 +35,13 @@ import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; -import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; @@ -110,8 +112,18 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class); private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(ElasticsearchInternalService.class); + /** + * Fix for https://github.com/elastic/elasticsearch/issues/124675 + * In 8.13.0 we transitioned from model_version to model_id. Any elser inference endpoints created prior to 8.13.0 will still use + * service_settings.model_version. + */ + private static final String OLD_MODEL_ID_FIELD_NAME = "model_version"; + + private final Settings settings; + public ElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { super(context); + this.settings = context.settings(); } // for testing @@ -120,6 +132,7 @@ public ElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFa Consumer> platformArch ) { super(context, platformArch); + this.settings = context.settings(); } @Override @@ -484,6 +497,8 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + migrateModelVersionToModelId(serviceSettingsMap); + ChunkingSettings chunkingSettings = null; if (TaskType.TEXT_EMBEDDING.equals(taskType) || TaskType.SPARSE_EMBEDDING.equals(taskType)) { chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS)); @@ -491,7 +506,9 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M String modelId = (String) serviceSettingsMap.get(MODEL_ID); if (modelId == null) { - throw new IllegalArgumentException("Error parsing request config, model id is missing"); + throw new IllegalArgumentException( + Strings.format("Error parsing request config, model id is missing for inference id: %s", inferenceEntityId) + ); } if (MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId)) { @@ -531,6 +548,18 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M } } + /** + * Fix for https://github.com/elastic/elasticsearch/issues/124675 + * In 8.13.0 we transitioned from model_version to model_id. Any elser inference endpoints created prior to 8.13.0 will still use + * service_settings.model_version. We need to look for that key and migrate it to model_id. + */ + private void migrateModelVersionToModelId(Map serviceSettingsMap) { + if (serviceSettingsMap.containsKey(OLD_MODEL_ID_FIELD_NAME)) { + String modelId = ServiceUtils.removeAsType(serviceSettingsMap, OLD_MODEL_ID_FIELD_NAME, String.class); + serviceSettingsMap.put(ElserInternalServiceSettings.MODEL_ID, modelId); + } + } + @Override public void checkModelConfig(Model model, ActionListener listener) { if (model instanceof CustomElandEmbeddingModel elandModel && elandModel.getTaskType() == TaskType.TEXT_EMBEDDING) { @@ -562,6 +591,7 @@ private static CustomElandEmbeddingModel updateModelWithEmbeddingDetails(CustomE model.getServiceSettings().getNumThreads(), model.getServiceSettings().modelId(), model.getServiceSettings().getAdaptiveAllocationsSettings(), + model.getServiceSettings().getDeploymentId(), embeddingSize, model.getServiceSettings().similarity(), model.getServiceSettings().elementType() @@ -842,12 +872,21 @@ public void updateModelsWithDynamicFields(List models, ActionListener(); + // if ML is disabled, do not update Deployment Stats (there won't be changes) + if (XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) == false) { + listener.onResponse(models); + return; + } + + var modelsByDeploymentIds = new HashMap>(); for (var model : models) { assert model instanceof ElasticsearchInternalModel; if (model instanceof ElasticsearchInternalModel esModel) { - modelsByDeploymentIds.put(esModel.mlNodeDeploymentId(), esModel); + modelsByDeploymentIds.merge(esModel.mlNodeDeploymentId(), new ArrayList<>(List.of(esModel)), (a, b) -> { + a.addAll(b); + return a; + }); } else { listener.onFailure( new ElasticsearchStatusException( @@ -866,10 +905,13 @@ public void updateModelsWithDynamicFields(List models, ActionListener { for (var deploymentStats : stats.getStats().results()) { - var model = modelsByDeploymentIds.get(deploymentStats.getDeploymentId()); - model.updateNumAllocations(deploymentStats.getNumberOfAllocations()); + var modelsForDeploymentId = modelsByDeploymentIds.get(deploymentStats.getDeploymentId()); + modelsForDeploymentId.forEach(model -> model.updateNumAllocations(deploymentStats.getNumberOfAllocations())); } - listener.onResponse(new ArrayList<>(modelsByDeploymentIds.values())); + var updatedModels = new ArrayList(); + modelsByDeploymentIds.values().forEach(updatedModels::addAll); + + listener.onResponse(updatedModels); }, e -> { logger.warn("Get deployment stats failed, cannot update the endpoint's number of allocations", e); // continue with the original response @@ -902,12 +944,7 @@ private List defaultConfigs(boolean useLinuxOptimizedModel) { DEFAULT_ELSER_ID, TaskType.SPARSE_EMBEDDING, NAME, - new ElserInternalServiceSettings( - null, - 1, - useLinuxOptimizedModel ? ELSER_V2_MODEL_LINUX_X86 : ELSER_V2_MODEL, - new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32) - ), + ElserInternalServiceSettings.defaultEndpointSettings(useLinuxOptimizedModel), ElserMlNodeTaskSettings.DEFAULT, ChunkingSettingsBuilder.DEFAULT_SETTINGS ); @@ -915,19 +952,14 @@ private List defaultConfigs(boolean useLinuxOptimizedModel) { DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, NAME, - new MultilingualE5SmallInternalServiceSettings( - null, - 1, - useLinuxOptimizedModel ? MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 : MULTILINGUAL_E5_SMALL_MODEL_ID, - new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32) - ), + MultilingualE5SmallInternalServiceSettings.defaultEndpointSettings(useLinuxOptimizedModel), ChunkingSettingsBuilder.DEFAULT_SETTINGS ); var defaultRerank = new ElasticRerankerModel( DEFAULT_RERANK_ID, TaskType.RERANK, NAME, - new ElasticRerankerServiceSettings(null, 1, RERANKER_ID, new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32)), + ElasticRerankerServiceSettings.defaultEndpointSettings(), RerankTaskSettings.DEFAULT_SETTINGS ); return List.of(defaultElser, defaultE5, defaultRerank); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index 244108edc3dd4..98730f33d10f9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.xcontent.ToXContentObject; @@ -108,25 +109,12 @@ protected static ElasticsearchInternalServiceSettings.Builder fromMap( .setDeploymentId(deploymentId); } - public ElasticsearchInternalServiceSettings( - Integer numAllocations, - int numThreads, - String modelId, - AdaptiveAllocationsSettings adaptiveAllocationsSettings - ) { - this.numAllocations = numAllocations; - this.numThreads = numThreads; - this.modelId = Objects.requireNonNull(modelId); - this.adaptiveAllocationsSettings = adaptiveAllocationsSettings; - this.deploymentId = null; - } - public ElasticsearchInternalServiceSettings( Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings, - String deploymentId + @Nullable String deploymentId ) { this.numAllocations = numAllocations; this.numThreads = numThreads; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java index da9164bf3f288..b94b9feb8a049 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java @@ -9,14 +9,14 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.inference.MinimalServiceSettings; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import java.io.IOException; -import java.util.Arrays; -import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL_LINUX_X86; public class ElserInternalServiceSettings extends ElasticsearchInternalServiceSettings { @@ -26,37 +26,26 @@ public static MinimalServiceSettings minimalServiceSettings() { return MinimalServiceSettings.sparseEmbedding(); } - public static Builder fromRequestMap(Map map) { - ValidationException validationException = new ValidationException(); - var baseSettings = ElasticsearchInternalServiceSettings.fromMap(map, validationException); - - String modelId = baseSettings.getModelId(); - if (modelId != null && ElserModels.isValidModel(modelId) == false) { - var ve = new ValidationException(); - ve.addValidationError( - "Unknown ELSER model ID [" + modelId + "]. Valid models are " + Arrays.toString(ElserModels.VALID_ELSER_MODEL_IDS.toArray()) - ); - throw ve; - } - - if (validationException.validationErrors().isEmpty() == false) { - throw validationException; - } - - return baseSettings; + public static ElserInternalServiceSettings defaultEndpointSettings(boolean useLinuxOptimizedModel) { + return new ElserInternalServiceSettings( + null, + 1, + useLinuxOptimizedModel ? ELSER_V2_MODEL_LINUX_X86 : ELSER_V2_MODEL, + new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32) + ); } public ElserInternalServiceSettings(ElasticsearchInternalServiceSettings other) { super(other); } - public ElserInternalServiceSettings( + private ElserInternalServiceSettings( Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings ) { - this(new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId, adaptiveAllocationsSettings)); + this(new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId, adaptiveAllocationsSettings, null)); } public ElserInternalServiceSettings(StreamInput in) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java index 317cc48172fca..45d52d3c8deaa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java @@ -18,6 +18,9 @@ import java.util.Arrays; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86; + public class MultilingualE5SmallInternalServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "multilingual_e5_small_service_settings"; @@ -29,17 +32,26 @@ public static MinimalServiceSettings minimalServiceSettings() { return MinimalServiceSettings.textEmbedding(DIMENSIONS, SIMILARITY, DenseVectorFieldMapper.ElementType.FLOAT); } + public static MultilingualE5SmallInternalServiceSettings defaultEndpointSettings(boolean useLinuxOptimizedModel) { + return new MultilingualE5SmallInternalServiceSettings( + null, + 1, + useLinuxOptimizedModel ? MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 : MULTILINGUAL_E5_SMALL_MODEL_ID, + new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32) + ); + } + public MultilingualE5SmallInternalServiceSettings(ElasticsearchInternalServiceSettings other) { super(other); } - public MultilingualE5SmallInternalServiceSettings( + MultilingualE5SmallInternalServiceSettings( Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings ) { - super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); + super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings, null); } public MultilingualE5SmallInternalServiceSettings(StreamInput in) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 205cc545a23f0..66e286c9098db 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -282,10 +282,7 @@ protected void doInfer( ) { if (model instanceof GoogleAiStudioCompletionModel completionModel) { var requestManager = new GoogleAiStudioCompletionRequestManager(completionModel, getServiceComponents().threadPool()); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - completionModel.uri(inputs.stream()), - "Google AI Studio completion" - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google AI Studio completion"); var action = new SingleInputSenderExecutableAction( getSender(), requestManager, @@ -299,7 +296,7 @@ protected void doInfer( getServiceComponents().truncator(), getServiceComponents().threadPool() ); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(embeddingsModel.uri(), "Google AI Studio embeddings"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google AI Studio embeddings"); var action = new SenderExecutableAction(getSender(), requestManager, failedToSendRequestErrorMessage); action.execute(inputs, timeout, listener); } else { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java index 55397b2398d39..3e921f669e864 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java @@ -327,6 +327,8 @@ public static InferenceServiceConfiguration get() { () -> { var configurationMap = new HashMap(); + // TODO whether the model ID is required or not depends on the task type + // For rerank it is optional, for text_embedding it is required configurationMap.put( MODEL_ID, new SettingsConfiguration.Builder(supportedTaskTypes).setDescription("ID of the LLM you're using.") diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxModel.java index 4f0b425cdaa51..09706f70e3684 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxModel.java @@ -12,6 +12,7 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.ibmwatsonx.IbmWatsonxActionVisitor; @@ -38,6 +39,12 @@ public IbmWatsonxModel(IbmWatsonxModel model, ServiceSettings serviceSettings) { rateLimitServiceSettings = model.rateLimitServiceSettings(); } + public IbmWatsonxModel(IbmWatsonxModel model, TaskSettings taskSettings) { + super(model, taskSettings); + + rateLimitServiceSettings = model.rateLimitServiceSettings(); + } + public abstract ExecutableAction accept(IbmWatsonxActionVisitor creator, Map taskSettings, InputType inputType); public IbmWatsonxRateLimitServiceSettings rateLimitServiceSettings() { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java index 477225f00d22b..3fa423c2dae19 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModel; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; @@ -138,6 +139,15 @@ private static IbmWatsonxModel createModel( secretSettings, context ); + case RERANK -> new IbmWatsonxRerankModel( + inferenceEntityId, + taskType, + NAME, + serviceSettings, + taskSettings, + secretSettings, + context + ); default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST); }; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankModel.java new file mode 100644 index 0000000000000..cb4c509d88c2b --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankModel.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank; + +import org.apache.http.client.utils.URIBuilder; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.ibmwatsonx.IbmWatsonxActionVisitor; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxModel; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxUtils.ML; +import static org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxUtils.RERANKS; +import static org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxUtils.TEXT; +import static org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxUtils.V1; + +public class IbmWatsonxRerankModel extends IbmWatsonxModel { + public static IbmWatsonxRerankModel of(IbmWatsonxRerankModel model, Map taskSettings) { + var requestTaskSettings = IbmWatsonxRerankTaskSettings.fromMap(taskSettings); + return new IbmWatsonxRerankModel(model, IbmWatsonxRerankTaskSettings.of(model.getTaskSettings(), requestTaskSettings)); + } + + public IbmWatsonxRerankModel( + String modelId, + TaskType taskType, + String service, + Map serviceSettings, + Map taskSettings, + @Nullable Map secrets, + ConfigurationParseContext context + ) { + this( + modelId, + taskType, + service, + IbmWatsonxRerankServiceSettings.fromMap(serviceSettings, context), + IbmWatsonxRerankTaskSettings.fromMap(taskSettings), + DefaultSecretSettings.fromMap(secrets) + ); + } + + // should only be used for testing + IbmWatsonxRerankModel( + String modelId, + TaskType taskType, + String service, + IbmWatsonxRerankServiceSettings serviceSettings, + IbmWatsonxRerankTaskSettings taskSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + super( + new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secretSettings), + serviceSettings + ); + } + + private IbmWatsonxRerankModel(IbmWatsonxRerankModel model, IbmWatsonxRerankTaskSettings taskSettings) { + super(model, taskSettings); + } + + @Override + public IbmWatsonxRerankServiceSettings getServiceSettings() { + return (IbmWatsonxRerankServiceSettings) super.getServiceSettings(); + } + + @Override + public IbmWatsonxRerankTaskSettings getTaskSettings() { + return (IbmWatsonxRerankTaskSettings) super.getTaskSettings(); + } + + @Override + public DefaultSecretSettings getSecretSettings() { + return (DefaultSecretSettings) super.getSecretSettings(); + } + + public URI uri() { + URI uri; + try { + uri = buildUri(this.getServiceSettings().uri().toString(), this.getServiceSettings().apiVersion()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + return uri; + } + + /** + * Accepts a visitor to create an executable action. The returned action will not return documents in the response. + * @param visitor _ + * @param taskSettings _ + * @param inputType ignored for rerank task + * @return the rerank action + */ + @Override + public ExecutableAction accept(IbmWatsonxActionVisitor visitor, Map taskSettings, InputType inputType) { + return visitor.create(this, taskSettings); + } + + public static URI buildUri(String uri, String apiVersion) throws URISyntaxException { + return new URIBuilder().setScheme("https") + .setHost(uri) + .setPathSegments(ML, V1, TEXT, RERANKS) + .setParameter("version", apiVersion) + .build(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankServiceSettings.java new file mode 100644 index 0000000000000..969622f9ba54f --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankServiceSettings.java @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxRateLimitServiceSettings; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxService; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; +import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.convertToUri; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createUri; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; +import static org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxServiceFields.API_VERSION; +import static org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxServiceFields.PROJECT_ID; + +public class IbmWatsonxRerankServiceSettings extends FilteredXContentObject implements ServiceSettings, IbmWatsonxRateLimitServiceSettings { + public static final String NAME = "ibm_watsonx_rerank_service_settings"; + + /** + * Rate limits are defined at + * Watson Machine Learning plans. + * For Lite plan, you've 120 requests per minute. + */ + private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(120); + + public static IbmWatsonxRerankServiceSettings fromMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + + String url = extractRequiredString(map, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + String apiVersion = extractRequiredString(map, API_VERSION, ModelConfigurations.SERVICE_SETTINGS, validationException); + + String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + String projectId = extractRequiredString(map, PROJECT_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + + RateLimitSettings rateLimitSettings = RateLimitSettings.of( + map, + DEFAULT_RATE_LIMIT_SETTINGS, + validationException, + IbmWatsonxService.NAME, + context + ); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new IbmWatsonxRerankServiceSettings(uri, apiVersion, modelId, projectId, rateLimitSettings); + } + + private final URI uri; + + private final String apiVersion; + + private final String modelId; + + private final String projectId; + + private final RateLimitSettings rateLimitSettings; + + public IbmWatsonxRerankServiceSettings( + URI uri, + String apiVersion, + String modelId, + String projectId, + @Nullable RateLimitSettings rateLimitSettings + ) { + this.uri = uri; + this.apiVersion = apiVersion; + this.projectId = projectId; + this.modelId = modelId; + this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS); + } + + public IbmWatsonxRerankServiceSettings(StreamInput in) throws IOException { + this.uri = createUri(in.readString()); + this.apiVersion = in.readString(); + this.modelId = in.readString(); + this.projectId = in.readString(); + this.rateLimitSettings = new RateLimitSettings(in); + + } + + public URI uri() { + return uri; + } + + public String apiVersion() { + return apiVersion; + } + + @Override + public String modelId() { + return modelId; + } + + public String projectId() { + return projectId; + } + + @Override + public RateLimitSettings rateLimitSettings() { + return rateLimitSettings; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + toXContentFragmentOfExposedFields(builder, params); + + builder.endObject(); + return builder; + } + + @Override + protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + builder.field(URL, uri.toString()); + + builder.field(API_VERSION, apiVersion); + + builder.field(MODEL_ID, modelId); + + builder.field(PROJECT_ID, projectId); + + rateLimitSettings.toXContent(builder, params); + + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ML_INFERENCE_IBM_WATSONX_RERANK_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(uri.toString()); + out.writeString(apiVersion); + + out.writeString(modelId); + out.writeString(projectId); + + rateLimitSettings.writeTo(out); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + IbmWatsonxRerankServiceSettings that = (IbmWatsonxRerankServiceSettings) object; + return Objects.equals(uri, that.uri) + && Objects.equals(apiVersion, that.apiVersion) + && Objects.equals(modelId, that.modelId) + && Objects.equals(projectId, that.projectId) + && Objects.equals(rateLimitSettings, that.rateLimitSettings); + } + + @Override + public int hashCode() { + return Objects.hash(uri, apiVersion, modelId, projectId, rateLimitSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankTaskSettings.java new file mode 100644 index 0000000000000..12f4b8f6fa33e --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankTaskSettings.java @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalBoolean; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; + +public class IbmWatsonxRerankTaskSettings implements TaskSettings { + + public static final String NAME = "ibm_watsonx_rerank_task_settings"; + public static final String RETURN_DOCUMENTS = "return_documents"; + public static final String TOP_N_DOCS_ONLY = "top_n"; + public static final String TRUNCATE_INPUT_TOKENS = "truncate_input_tokens"; + + static final IbmWatsonxRerankTaskSettings EMPTY_SETTINGS = new IbmWatsonxRerankTaskSettings(null, null, null); + + public static IbmWatsonxRerankTaskSettings fromMap(Map map) { + ValidationException validationException = new ValidationException(); + + if (map == null || map.isEmpty()) { + return EMPTY_SETTINGS; + } + + Boolean returnDocuments = extractOptionalBoolean(map, RETURN_DOCUMENTS, validationException); + Integer topNDocumentsOnly = extractOptionalPositiveInteger( + map, + TOP_N_DOCS_ONLY, + ModelConfigurations.TASK_SETTINGS, + validationException + ); + Integer truncateInputTokens = extractOptionalPositiveInteger( + map, + TRUNCATE_INPUT_TOKENS, + ModelConfigurations.TASK_SETTINGS, + validationException + ); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return of(topNDocumentsOnly, returnDocuments, truncateInputTokens); + } + + /** + * Creates a new {@link IbmWatsonxRerankTaskSettings} + * by preferring non-null fields from the request settings over the original settings. + * + * @param originalSettings the settings stored as part of the inference entity configuration + * @param requestTaskSettings the settings passed in within the task_settings field of the request + * @return a constructed {@link IbmWatsonxRerankTaskSettings} + */ + public static IbmWatsonxRerankTaskSettings of( + IbmWatsonxRerankTaskSettings originalSettings, + IbmWatsonxRerankTaskSettings requestTaskSettings + ) { + return new IbmWatsonxRerankTaskSettings( + requestTaskSettings.getTopNDocumentsOnly() != null + ? requestTaskSettings.getTopNDocumentsOnly() + : originalSettings.getTopNDocumentsOnly(), + requestTaskSettings.getReturnDocuments() != null + ? requestTaskSettings.getReturnDocuments() + : originalSettings.getReturnDocuments(), + requestTaskSettings.getTruncateInputTokens() != null + ? requestTaskSettings.getTruncateInputTokens() + : originalSettings.getTruncateInputTokens() + ); + } + + public static IbmWatsonxRerankTaskSettings of(Integer topNDocumentsOnly, Boolean returnDocuments, Integer maxChunksPerDoc) { + return new IbmWatsonxRerankTaskSettings(topNDocumentsOnly, returnDocuments, maxChunksPerDoc); + } + + private final Integer topNDocumentsOnly; + private final Boolean returnDocuments; + private final Integer truncateInputTokens; + + public IbmWatsonxRerankTaskSettings(StreamInput in) throws IOException { + this(in.readOptionalInt(), in.readOptionalBoolean(), in.readOptionalInt()); + } + + public IbmWatsonxRerankTaskSettings( + @Nullable Integer topNDocumentsOnly, + @Nullable Boolean doReturnDocuments, + @Nullable Integer truncateInputTokens + ) { + this.topNDocumentsOnly = topNDocumentsOnly; + this.returnDocuments = doReturnDocuments; + this.truncateInputTokens = truncateInputTokens; + } + + @Override + public boolean isEmpty() { + return topNDocumentsOnly == null && returnDocuments == null && truncateInputTokens == null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (topNDocumentsOnly != null) { + builder.field(TOP_N_DOCS_ONLY, topNDocumentsOnly); + } + if (returnDocuments != null) { + builder.field(RETURN_DOCUMENTS, returnDocuments); + } + if (truncateInputTokens != null) { + builder.field(TRUNCATE_INPUT_TOKENS, truncateInputTokens); + } + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ML_INFERENCE_IBM_WATSONX_RERANK_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalInt(topNDocumentsOnly); + out.writeOptionalBoolean(returnDocuments); + out.writeOptionalInt(truncateInputTokens); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IbmWatsonxRerankTaskSettings that = (IbmWatsonxRerankTaskSettings) o; + return Objects.equals(returnDocuments, that.returnDocuments) + && Objects.equals(topNDocumentsOnly, that.topNDocumentsOnly) + && Objects.equals(truncateInputTokens, that.truncateInputTokens); + } + + @Override + public int hashCode() { + return Objects.hash(returnDocuments, topNDocumentsOnly, truncateInputTokens); + } + + public static String invalidInputTypeMessage(InputType inputType) { + return Strings.format("received invalid input type value [%s]", inputType.toString()); + } + + public Boolean getDoesReturnDocuments() { + return returnDocuments; + } + + public Integer getTopNDocumentsOnly() { + return topNDocumentsOnly; + } + + public Boolean getReturnDocuments() { + return returnDocuments; + } + + public Integer getTruncateInputTokens() { + return truncateInputTokens; + } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + IbmWatsonxRerankTaskSettings updatedSettings = IbmWatsonxRerankTaskSettings.fromMap(new HashMap<>(newSettings)); + return IbmWatsonxRerankTaskSettings.of(this, updatedSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java index 7ad70fc88054d..37add1e264704 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java @@ -25,6 +25,7 @@ import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; @@ -49,6 +50,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -339,6 +341,33 @@ public static InferenceServiceConfiguration get() { () -> { var configurationMap = new HashMap(); + configurationMap.put( + JinaAIServiceSettings.MODEL_ID, + new SettingsConfiguration.Builder(supportedTaskTypes).setDescription( + "The name of the model to use for the inference task." + ) + .setLabel("Model ID") + .setRequired(true) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.STRING) + .build() + ); + + configurationMap.put( + DIMENSIONS, + new SettingsConfiguration.Builder(EnumSet.of(TaskType.TEXT_EMBEDDING)).setDescription( + "The number of dimensions the resulting embeddings should have. For more information refer to " + + "https://api.jina.ai/redoc#tag/embeddings/operation/create_embedding_v1_embeddings_post." + ) + .setLabel("Dimensions") + .setRequired(false) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.INTEGER) + .build() + ); + configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration(supportedTaskTypes)); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration(supportedTaskTypes)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 0ce5bc801b59f..8f8dba02977f6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -56,8 +56,8 @@ import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionCreator.COMPLETION_ERROR_PREFIX; +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; -import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -296,7 +296,7 @@ public void doUnifiedCompletionInfer( var overriddenModel = OpenAiChatCompletionModel.of(openAiModel, inputs.getRequest()); var requestCreator = OpenAiUnifiedCompletionRequestManager.of(overriddenModel, getServiceComponents().threadPool()); - var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getServiceSettings().uri(), COMPLETION_ERROR_PREFIX); + var errorMessage = constructFailedToSendRequestMessage(COMPLETION_ERROR_PREFIX); var action = new SenderExecutableAction(getSender(), requestCreator, errorMessage); action.execute(inputs, timeout, listener); @@ -440,19 +440,16 @@ public static InferenceServiceConfiguration get() { ); configurationMap.put( - URL, - new SettingsConfiguration.Builder(SUPPORTED_TASK_TYPES_FOR_SERVICES_API).setDefaultValue( - "https://api.openai.com/v1/chat/completions" + DIMENSIONS, + new SettingsConfiguration.Builder(EnumSet.of(TaskType.TEXT_EMBEDDING)).setDescription( + "The number of dimensions the resulting embeddings should have. For more information refer to " + + "https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-dimensions." ) - .setDescription( - "The OpenAI API endpoint URL. For more information on the URL, refer to the " - + "https://platform.openai.com/docs/api-reference." - ) - .setLabel("URL") - .setRequired(true) + .setLabel("Dimensions") + .setRequired(false) .setSensitive(false) .setUpdatable(false) - .setType(SettingsConfigurationFieldType.STRING) + .setType(SettingsConfigurationFieldType.INTEGER) .build() ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/ModelValidatorBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/ModelValidatorBuilder.java index e4de3d6beb800..1c4306c4edd46 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/ModelValidatorBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/ModelValidatorBuilder.java @@ -29,7 +29,7 @@ public static ModelValidator buildModelValidator(TaskType taskType) { case SPARSE_EMBEDDING, RERANK, ANY -> { return new SimpleModelValidator(new SimpleServiceIntegrationValidator()); } - default -> throw new IllegalArgumentException(Strings.format("Can't validate inference model of for task type %s ", taskType)); + default -> throw new IllegalArgumentException(Strings.format("Can't validate inference model for task type %s", taskType)); } } } diff --git a/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml index 41383d0b6736a..641f68985a710 100644 --- a/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/inference/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,35 @@ com.google.api.client: - set_https_connection_properties + - outbound_network +software.amazon.awssdk.utils: + - manage_threads + - outbound_network +# AmazonBedrockInferenceClient uses NettyNioAsyncHttpClient, so we grant network permissions (and thread permissions, +# as it is async) to the related modules +software.amazon.awssdk.http.nio.netty: + - manage_threads + - outbound_network +io.netty.common: + - outbound_network + - manage_threads + - files: + - path: "/etc/os-release" + mode: "read" + - path: "/usr/lib/os-release" + mode: "read" + - path: "/proc/sys/net/core/somaxconn" + mode: read +io.netty.transport: + - manage_threads + - outbound_network +# AWS Clients always try to access the credentials and config files, even if we configure otherwise +# This should be "fixed" (as in, it will handle SecurityException correctly) +# by https://github.com/aws/aws-sdk-java-v2/pull/5904. Once confirmed and libraries are updated, these could be removed. +software.amazon.awssdk.profiles: + - files: + - relative_path: .aws/credentials + relative_to: home + mode: read + - relative_path: .aws/config + relative_to: home + mode: read diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InferenceExceptionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InferenceExceptionTests.java new file mode 100644 index 0000000000000..ba9daf95d17c2 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InferenceExceptionTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import static org.hamcrest.CoreMatchers.equalTo; + +public class InferenceExceptionTests extends ESTestCase { + public void testWrapException() throws Exception { + ElasticsearchStatusException cause = new ElasticsearchStatusException("test", RestStatus.BAD_REQUEST); + InferenceException testException = new InferenceException("test wrapper", cause); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + testException.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + assertThat( + Strings.toString(builder), + equalTo( + "{\"type\":\"inference_exception\",\"reason\":\"test wrapper\"," + + "\"caused_by\":{\"type\":\"status_exception\",\"reason\":\"test\"}}" + ) + ); + assertThat(testException.status(), equalTo(RestStatus.BAD_REQUEST)); + } + + public void testNullCause() throws Exception { + InferenceException testException = new InferenceException("test exception", null); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + testException.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + assertThat(Strings.toString(builder), equalTo("{\"type\":\"inference_exception\",\"reason\":\"test exception\"}")); + assertThat(testException.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java index 1260b89034e6b..85300f24deea4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension; import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettings; import org.hamcrest.Matchers; import java.io.IOException; @@ -59,6 +60,8 @@ public final class Utils { + public static final TimeValue TIMEOUT = TimeValue.timeValueSeconds(30); + private Utils() { throw new UnsupportedOperationException("Utils is a utility class and should not be instantiated"); } @@ -76,7 +79,8 @@ public static ClusterService mockClusterService(Settings settings) { ThrottlerManager.getSettingsDefinitions(), RetrySettings.getSettingsDefinitions(), Truncator.getSettingsDefinitions(), - RequestExecutorServiceSettings.getSettingsDefinitions() + RequestExecutorServiceSettings.getSettingsDefinitions(), + ElasticInferenceServiceSettings.getSettingsDefinitions() ).flatMap(Collection::stream).collect(Collectors.toSet()); var cSettings = new ClusterSettings(settings, registeredSettings); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java index 4fa0a1ec49c74..56966ca40c478 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.inference.action; -import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.node.NodeClient; @@ -28,7 +28,7 @@ import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; -import org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculator; +import org.elasticsearch.xpack.inference.common.InferenceServiceRateLimitCalculator; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; import org.junit.Before; @@ -47,9 +47,9 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.assertArg; -import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -64,7 +64,7 @@ public abstract class BaseTransportInferenceActionTestCase createAction( InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, StreamingTaskManager streamingTaskManager, - InferenceServiceNodeLocalRateLimitCalculator inferenceServiceNodeLocalRateLimitCalculator, + InferenceServiceRateLimitCalculator inferenceServiceNodeLocalRateLimitCalculator, NodeClient nodeClient, ThreadPool threadPool ); @@ -127,8 +127,7 @@ public void testMetricsAfterModelRegistryError() { return null; }).when(modelRegistry).getModelWithSecrets(any(), any()); - var listener = doExecute(taskType); - verify(listener).onFailure(same(expectedException)); + doExecute(taskType); verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { assertThat(attributes.get("service"), nullValue()); @@ -148,7 +147,13 @@ protected ActionListener doExecute(TaskType taskType, when(request.getInferenceEntityId()).thenReturn(inferenceId); when(request.getTaskType()).thenReturn(taskType); when(request.isStreaming()).thenReturn(stream); - ActionListener listener = mock(); + ActionListener listener = spy(new ActionListener<>() { + @Override + public void onResponse(InferenceAction.Response o) {} + + @Override + public void onFailure(Exception e) {} + }); action.doExecute(mock(), request, listener); return listener; } @@ -161,9 +166,9 @@ public void testMetricsAfterMissingService() { var listener = doExecute(taskType); verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat(e, isA(ElasticsearchException.class)); assertThat(e.getMessage(), is("Unknown service [" + serviceId + "] for model [" + inferenceId + "]. ")); - assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + assertThat(((ElasticsearchException) e).status(), is(RestStatus.BAD_REQUEST)); })); verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { assertThat(attributes.get("service"), is(serviceId)); @@ -192,7 +197,7 @@ public void testMetricsAfterUnknownTaskType() { var listener = doExecute(requestTaskType); verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat(e, isA(ElasticsearchException.class)); assertThat( e.getMessage(), is( @@ -203,7 +208,7 @@ public void testMetricsAfterUnknownTaskType() { + "]" ) ); - assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + assertThat(((ElasticsearchException) e).status(), is(RestStatus.BAD_REQUEST)); })); verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { assertThat(attributes.get("service"), is(serviceId)); @@ -221,7 +226,6 @@ public void testMetricsAfterInferError() { var listener = doExecute(taskType); - verify(listener).onFailure(same(expectedException)); verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { assertThat(attributes.get("service"), is(serviceId)); assertThat(attributes.get("task_type"), is(taskType.toString())); @@ -239,8 +243,8 @@ public void testMetricsAfterStreamUnsupported() { var listener = doExecute(taskType, true); verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); - var ese = (ElasticsearchStatusException) e; + assertThat(e, isA(ElasticsearchException.class)); + var ese = (ElasticsearchException) e; assertThat(ese.getMessage(), is("Streaming is not allowed for service [" + serviceId + "].")); assertThat(ese.status(), is(expectedStatus)); })); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointActionTests.java new file mode 100644 index 0000000000000..b015eade0ecfe --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointActionTests.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.junit.After; +import org.junit.Before; + +import java.util.Map; +import java.util.Optional; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportDeleteInferenceEndpointActionTests extends ESTestCase { + + private static final TimeValue TIMEOUT = TimeValue.timeValueSeconds(30); + + private TransportDeleteInferenceEndpointAction action; + private ThreadPool threadPool; + private ModelRegistry mockModelRegistry; + private InferenceServiceRegistry mockInferenceServiceRegistry; + + @Before + public void setUp() throws Exception { + super.setUp(); + threadPool = createThreadPool(inferenceUtilityPool()); + mockModelRegistry = mock(ModelRegistry.class); + mockInferenceServiceRegistry = mock(InferenceServiceRegistry.class); + action = new TransportDeleteInferenceEndpointAction( + mock(TransportService.class), + mock(ClusterService.class), + threadPool, + mock(ActionFilters.class), + mockModelRegistry, + mockInferenceServiceRegistry + ); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + terminate(threadPool); + } + + public void testFailsToDelete_ADefaultEndpoint_WithoutPassingForceQueryParameter() { + doAnswer(invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(1); + listener.onResponse(new UnparsedModel("model_id", TaskType.COMPLETION, "service", Map.of(), Map.of())); + return Void.TYPE; + }).when(mockModelRegistry).getModel(anyString(), any()); + when(mockModelRegistry.containsDefaultConfigId(anyString())).thenReturn(true); + + var listener = new PlainActionFuture(); + + action.masterOperation( + mock(Task.class), + new DeleteInferenceEndpointAction.Request("model-id", TaskType.COMPLETION, false, false), + ClusterState.EMPTY_STATE, + listener + ); + + var exception = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + exception.getMessage(), + is("[model-id] is a reserved inference endpoint. Use the force=true query parameter to delete the inference endpoint.") + ); + } + + public void testDeletesDefaultEndpoint_WhenForceIsTrue() { + doAnswer(invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(1); + listener.onResponse(new UnparsedModel("model_id", TaskType.COMPLETION, "service", Map.of(), Map.of())); + return Void.TYPE; + }).when(mockModelRegistry).getModel(anyString(), any()); + when(mockModelRegistry.containsDefaultConfigId(anyString())).thenReturn(true); + doAnswer(invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(1); + listener.onResponse(true); + return Void.TYPE; + }).when(mockModelRegistry).deleteModel(anyString(), any()); + + var mockService = mock(InferenceService.class); + doAnswer(invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(1); + listener.onResponse(true); + return Void.TYPE; + }).when(mockService).stop(any(), any()); + + when(mockInferenceServiceRegistry.getService(anyString())).thenReturn(Optional.of(mockService)); + + var listener = new PlainActionFuture(); + + action.masterOperation( + mock(Task.class), + new DeleteInferenceEndpointAction.Request("model-id", TaskType.COMPLETION, true, false), + ClusterState.EMPTY_STATE, + listener + ); + + var response = listener.actionGet(TIMEOUT); + + assertTrue(response.isAcknowledged()); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionProxyTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionProxyTests.java new file mode 100644 index 0000000000000..a9e6ec55a6224 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionProxyTests.java @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TransportInferenceActionProxyTests extends ESTestCase { + private Client client; + private ThreadPool threadPool; + private TransportInferenceActionProxy action; + private ModelRegistry modelRegistry; + + @Before + public void setUp() throws Exception { + super.setUp(); + client = mock(Client.class); + threadPool = new TestThreadPool("test"); + when(client.threadPool()).thenReturn(threadPool); + modelRegistry = mock(ModelRegistry.class); + + action = new TransportInferenceActionProxy(mock(TransportService.class), mock(ActionFilters.class), modelRegistry, client); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + terminate(threadPool); + } + + public void testExecutesAUnifiedCompletionRequest_WhenTaskTypeIsChatCompletion_InRequest() { + String requestJson = """ + { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + { + "text": "some text", + "type": "string" + } + ] + } + ] + } + """; + + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) mock(ActionListener.class); + var request = new InferenceActionProxy.Request( + TaskType.CHAT_COMPLETION, + "id", + new BytesArray(requestJson), + XContentType.JSON, + TimeValue.ONE_MINUTE, + true + ); + + action.doExecute(mock(Task.class), request, listener); + + verify(client, times(1)).execute(eq(UnifiedCompletionAction.INSTANCE), any(), any()); + } + + public void testExecutesAUnifiedCompletionRequest_WhenTaskTypeIsChatCompletion_FromStorage() { + String requestJson = """ + { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + { + "text": "some text", + "type": "string" + } + ] + } + ] + } + """; + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse( + new UnparsedModel("id", TaskType.CHAT_COMPLETION, "service", Collections.emptyMap(), Collections.emptyMap()) + ); + + return Void.TYPE; + }).when(modelRegistry).getModelWithSecrets(any(), any()); + + var listener = new PlainActionFuture(); + var request = new InferenceActionProxy.Request( + TaskType.ANY, + "id", + new BytesArray(requestJson), + XContentType.JSON, + TimeValue.ONE_MINUTE, + true + ); + + action.doExecute(mock(Task.class), request, listener); + + verify(client, times(1)).execute(eq(UnifiedCompletionAction.INSTANCE), any(), any()); + } + + public void testExecutesAnInferenceAction_WhenTaskTypeIsCompletion_InRequest() { + String requestJson = """ + { + "input": ["some text"] + } + """; + + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) mock(ActionListener.class); + var request = new InferenceActionProxy.Request( + TaskType.COMPLETION, + "id", + new BytesArray(requestJson), + XContentType.JSON, + TimeValue.ONE_MINUTE, + true + ); + + action.doExecute(mock(Task.class), request, listener); + + verify(client, times(1)).execute(eq(InferenceAction.INSTANCE), any(), any()); + } + + public void testExecutesAnInferenceAction_WhenTaskTypeIsCompletion_FromStorage() { + String requestJson = """ + { + "input": ["some text"] + } + """; + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse(new UnparsedModel("id", TaskType.COMPLETION, "service", Collections.emptyMap(), Collections.emptyMap())); + + return Void.TYPE; + }).when(modelRegistry).getModelWithSecrets(any(), any()); + + var listener = new PlainActionFuture(); + var request = new InferenceActionProxy.Request( + TaskType.ANY, + "id", + new BytesArray(requestJson), + XContentType.JSON, + TimeValue.ONE_MINUTE, + true + ); + + action.doExecute(mock(Task.class), request, listener); + + verify(client, times(1)).execute(eq(InferenceAction.INSTANCE), any(), any()); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java index e71d15dbe0420..3129f0865a249 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; -import org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculator; +import org.elasticsearch.xpack.inference.common.InferenceServiceRateLimitCalculator; import org.elasticsearch.xpack.inference.common.RateLimitAssignment; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; @@ -50,7 +50,7 @@ protected BaseTransportInferenceAction createAction( InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, StreamingTaskManager streamingTaskManager, - InferenceServiceNodeLocalRateLimitCalculator inferenceServiceNodeLocalRateLimitCalculator, + InferenceServiceRateLimitCalculator inferenceServiceNodeLocalRateLimitCalculator, NodeClient nodeClient, ThreadPool threadPool ) { @@ -77,7 +77,7 @@ public void testNoRerouting_WhenTaskTypeNotSupported() { TaskType unsupportedTaskType = TaskType.COMPLETION; mockService(listener -> listener.onResponse(mock())); - when(inferenceServiceNodeLocalRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, unsupportedTaskType)).thenReturn(false); + when(inferenceServiceRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, unsupportedTaskType)).thenReturn(false); var listener = doExecute(unsupportedTaskType); @@ -89,8 +89,8 @@ public void testNoRerouting_WhenTaskTypeNotSupported() { public void testNoRerouting_WhenNoGroupingCalculatedYet() { mockService(listener -> listener.onResponse(mock())); - when(inferenceServiceNodeLocalRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); - when(inferenceServiceNodeLocalRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(null); + when(inferenceServiceRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); + when(inferenceServiceRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(null); var listener = doExecute(taskType); @@ -102,8 +102,8 @@ public void testNoRerouting_WhenNoGroupingCalculatedYet() { public void testNoRerouting_WhenEmptyNodeList() { mockService(listener -> listener.onResponse(mock())); - when(inferenceServiceNodeLocalRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); - when(inferenceServiceNodeLocalRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn( + when(inferenceServiceRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); + when(inferenceServiceRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn( new RateLimitAssignment(List.of()) ); @@ -120,10 +120,10 @@ public void testRerouting_ToOtherNode() { // The local node is different to the "other-node" responsible for serviceId when(nodeClient.getLocalNodeId()).thenReturn("local-node"); - when(inferenceServiceNodeLocalRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); + when(inferenceServiceRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); // Requests for serviceId are always routed to "other-node" var assignment = new RateLimitAssignment(List.of(otherNode)); - when(inferenceServiceNodeLocalRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(assignment); + when(inferenceServiceRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(assignment); mockService(listener -> listener.onResponse(mock())); var listener = doExecute(taskType); @@ -141,9 +141,9 @@ public void testRerouting_ToLocalNode_WithoutGoingThroughTransportLayerAgain() { // The local node is the only one responsible for serviceId when(nodeClient.getLocalNodeId()).thenReturn(localNodeId); - when(inferenceServiceNodeLocalRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); + when(inferenceServiceRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); var assignment = new RateLimitAssignment(List.of(localNode)); - when(inferenceServiceNodeLocalRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(assignment); + when(inferenceServiceRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(assignment); mockService(listener -> listener.onResponse(mock())); var listener = doExecute(taskType); @@ -158,9 +158,9 @@ public void testRerouting_HandlesTransportException_FromOtherNode() { when(otherNode.getId()).thenReturn("other-node"); when(nodeClient.getLocalNodeId()).thenReturn("local-node"); - when(inferenceServiceNodeLocalRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); + when(inferenceServiceRateLimitCalculator.isTaskTypeReroutingSupported(serviceId, taskType)).thenReturn(true); var assignment = new RateLimitAssignment(List.of(otherNode)); - when(inferenceServiceNodeLocalRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(assignment); + when(inferenceServiceRateLimitCalculator.getRateLimitAssignment(serviceId, taskType)).thenReturn(assignment); mockService(listener -> listener.onResponse(mock())); @@ -173,6 +173,10 @@ public void testRerouting_HandlesTransportException_FromOtherNode() { var listener = doExecute(taskType); + // Verify request was rerouted + verify(transportService).sendRequest(same(otherNode), eq(InferenceAction.NAME), any(), any()); + // Verify local execution didn't happen + verify(listener, never()).onResponse(any()); // Verify exception was propagated from "other-node" to "local-node" verify(listener).onFailure(same(expectedException)); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java index 4ed69e5abe537..7dac6a1015aae 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.action; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.inference.InferenceServiceRegistry; @@ -17,8 +16,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; -import org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculator; +import org.elasticsearch.xpack.inference.common.InferenceServiceRateLimitCalculator; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; @@ -49,7 +49,7 @@ protected BaseTransportInferenceAction createAc InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, StreamingTaskManager streamingTaskManager, - InferenceServiceNodeLocalRateLimitCalculator inferenceServiceNodeLocalRateLimitCalculator, + InferenceServiceRateLimitCalculator inferenceServiceRateLimitCalculator, NodeClient nodeClient, ThreadPool threadPool ) { @@ -61,7 +61,7 @@ protected BaseTransportInferenceAction createAc serviceRegistry, inferenceStats, streamingTaskManager, - inferenceServiceNodeLocalRateLimitCalculator, + inferenceServiceRateLimitCalculator, nodeClient, threadPool ); @@ -81,12 +81,12 @@ public void testThrows_IncompatibleTaskTypeException_WhenUsingATextEmbeddingInfe var listener = doExecute(requestTaskType); verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat(e, isA(UnifiedChatCompletionException.class)); assertThat( e.getMessage(), is("Incompatible task_type for unified API, the requested type [" + requestTaskType + "] must be one of [chat_completion]") ); - assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + assertThat(((UnifiedChatCompletionException) e).status(), is(RestStatus.BAD_REQUEST)); })); verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { assertThat(attributes.get("service"), is(serviceId)); @@ -106,12 +106,12 @@ public void testThrows_IncompatibleTaskTypeException_WhenUsingRequestIsAny_Model var listener = doExecute(requestTaskType); verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat(e, isA(UnifiedChatCompletionException.class)); assertThat( e.getMessage(), is("Incompatible task_type for unified API, the requested type [" + requestTaskType + "] must be one of [chat_completion]") ); - assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + assertThat(((UnifiedChatCompletionException) e).status(), is(RestStatus.BAD_REQUEST)); })); verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { assertThat(attributes.get("service"), is(serviceId)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/UpdateInferenceModelActionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/UpdateInferenceModelActionRequestTests.java new file mode 100644 index 0000000000000..ab1e1d9c4cb23 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/UpdateInferenceModelActionRequestTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; +import org.elasticsearch.xpack.inference.InferenceNamedWriteablesProvider; + +import java.io.IOException; + +public class UpdateInferenceModelActionRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return UpdateInferenceModelAction.Request::new; + } + + @Override + protected UpdateInferenceModelAction.Request createTestInstance() { + return new UpdateInferenceModelAction.Request( + randomAlphaOfLength(5), + randomBytesReference(50), + randomFrom(XContentType.values()), + randomFrom(TaskType.values()), + randomTimeValue() + ); + } + + @Override + protected UpdateInferenceModelAction.Request mutateInstance(UpdateInferenceModelAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(InferenceNamedWriteablesProvider.getNamedWriteables()); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/UpdateInferenceModelActionResponseTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/UpdateInferenceModelActionResponseTests.java new file mode 100644 index 0000000000000..02208511d1484 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/UpdateInferenceModelActionResponseTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; +import org.elasticsearch.xpack.inference.InferenceNamedWriteablesProvider; +import org.elasticsearch.xpack.inference.ModelConfigurationsTests; + +import java.io.IOException; + +public class UpdateInferenceModelActionResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return UpdateInferenceModelAction.Response::new; + } + + @Override + protected UpdateInferenceModelAction.Response createTestInstance() { + return new UpdateInferenceModelAction.Response(ModelConfigurationsTests.createRandomInstance()); + } + + @Override + protected UpdateInferenceModelAction.Response mutateInstance(UpdateInferenceModelAction.Response instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(InferenceNamedWriteablesProvider.getNamedWriteables()); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java index 1fca17f77ad9a..fe135e0389576 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java @@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkItemRequest; @@ -40,6 +41,7 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -47,8 +49,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; +import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.mapper.SemanticTextField; import org.elasticsearch.xpack.inference.model.TestModel; import org.elasticsearch.xpack.inference.registry.ModelRegistry; @@ -113,7 +117,7 @@ public void tearDownThreadPool() throws Exception { @SuppressWarnings({ "unchecked", "rawtypes" }) public void testFilterNoop() throws Exception { - ShardBulkInferenceActionFilter filter = createFilter(threadPool, Map.of(), DEFAULT_BATCH_SIZE, useLegacyFormat); + ShardBulkInferenceActionFilter filter = createFilter(threadPool, Map.of(), DEFAULT_BATCH_SIZE, useLegacyFormat, true); CountDownLatch chainExecuted = new CountDownLatch(1); ActionFilterChain actionFilterChain = (task, action, request, listener) -> { try { @@ -136,6 +140,44 @@ public void testFilterNoop() throws Exception { awaitLatch(chainExecuted, 10, TimeUnit.SECONDS); } + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testLicenseInvalidForInference() throws InterruptedException { + StaticModel model = StaticModel.createRandomInstance(); + ShardBulkInferenceActionFilter filter = createFilter(threadPool, Map.of(), DEFAULT_BATCH_SIZE, useLegacyFormat, false); + CountDownLatch chainExecuted = new CountDownLatch(1); + ActionFilterChain actionFilterChain = (task, action, request, listener) -> { + try { + BulkShardRequest bulkShardRequest = (BulkShardRequest) request; + assertThat(bulkShardRequest.items().length, equalTo(1)); + + BulkItemResponse.Failure failure = bulkShardRequest.items()[0].getPrimaryResponse().getFailure(); + assertNotNull(failure); + assertThat(failure.getCause(), instanceOf(ElasticsearchSecurityException.class)); + assertThat( + failure.getMessage(), + containsString(org.elasticsearch.core.Strings.format("current license is non-compliant for [%s]", XPackField.INFERENCE)) + ); + } finally { + chainExecuted.countDown(); + } + + }; + ActionListener actionListener = mock(ActionListener.class); + Task task = mock(Task.class); + + Map inferenceFieldMap = Map.of( + "obj.field1", + new InferenceFieldMetadata("obj.field1", model.getInferenceEntityId(), new String[] { "obj.field1" }) + ); + BulkItemRequest[] items = new BulkItemRequest[1]; + items[0] = new BulkItemRequest(0, new IndexRequest("test").source("obj.field1", "Test")); + BulkShardRequest request = new BulkShardRequest(new ShardId("test", "test", 0), WriteRequest.RefreshPolicy.NONE, items); + request.setInferenceFieldMap(inferenceFieldMap); + + filter.apply(task, TransportShardBulkAction.ACTION_NAME, request, actionListener, actionFilterChain); + awaitLatch(chainExecuted, 10, TimeUnit.SECONDS); + } + @SuppressWarnings({ "unchecked", "rawtypes" }) public void testInferenceNotFound() throws Exception { StaticModel model = StaticModel.createRandomInstance(); @@ -143,7 +185,8 @@ public void testInferenceNotFound() throws Exception { threadPool, Map.of(model.getInferenceEntityId(), model), randomIntBetween(1, 10), - useLegacyFormat + useLegacyFormat, + true ); CountDownLatch chainExecuted = new CountDownLatch(1); ActionFilterChain actionFilterChain = (task, action, request, listener) -> { @@ -189,7 +232,8 @@ public void testItemFailures() throws Exception { threadPool, Map.of(model.getInferenceEntityId(), model), randomIntBetween(1, 10), - useLegacyFormat + useLegacyFormat, + true ); model.putResult("I am a failure", new ChunkedInferenceError(new IllegalArgumentException("boom"))); model.putResult("I am a success", randomChunkedInferenceEmbeddingSparse(List.of("I am a success"))); @@ -204,7 +248,9 @@ public void testItemFailures() throws Exception { assertNotNull(bulkShardRequest.items()[0].getPrimaryResponse()); assertTrue(bulkShardRequest.items()[0].getPrimaryResponse().isFailed()); BulkItemResponse.Failure failure = bulkShardRequest.items()[0].getPrimaryResponse().getFailure(); + assertThat(failure.getCause().getMessage(), containsString("Exception when running inference")); assertThat(failure.getCause().getCause().getMessage(), containsString("boom")); + assertThat(failure.getStatus(), is(RestStatus.BAD_REQUEST)); // item 1 is a success assertNull(bulkShardRequest.items()[1].getPrimaryResponse()); @@ -223,7 +269,9 @@ public void testItemFailures() throws Exception { assertNotNull(bulkShardRequest.items()[2].getPrimaryResponse()); assertTrue(bulkShardRequest.items()[2].getPrimaryResponse().isFailed()); failure = bulkShardRequest.items()[2].getPrimaryResponse().getFailure(); + assertThat(failure.getCause().getMessage(), containsString("Exception when running inference")); assertThat(failure.getCause().getCause().getMessage(), containsString("boom")); + assertThat(failure.getStatus(), is(RestStatus.BAD_REQUEST)); } finally { chainExecuted.countDown(); } @@ -255,7 +303,8 @@ public void testExplicitNull() throws Exception { threadPool, Map.of(model.getInferenceEntityId(), model), randomIntBetween(1, 10), - useLegacyFormat + useLegacyFormat, + true ); CountDownLatch chainExecuted = new CountDownLatch(1); @@ -344,7 +393,13 @@ public void testManyRandomDocs() throws Exception { modifiedRequests[id] = res[1]; } - ShardBulkInferenceActionFilter filter = createFilter(threadPool, inferenceModelMap, randomIntBetween(10, 30), useLegacyFormat); + ShardBulkInferenceActionFilter filter = createFilter( + threadPool, + inferenceModelMap, + randomIntBetween(10, 30), + useLegacyFormat, + true + ); CountDownLatch chainExecuted = new CountDownLatch(1); ActionFilterChain actionFilterChain = (task, action, request, listener) -> { try { @@ -379,7 +434,8 @@ private static ShardBulkInferenceActionFilter createFilter( ThreadPool threadPool, Map modelMap, int batchSize, - boolean useLegacyFormat + boolean useLegacyFormat, + boolean isLicenseValidForInference ) { ModelRegistry modelRegistry = mock(ModelRegistry.class); Answer unparsedModelAnswer = invocationOnMock -> { @@ -437,10 +493,14 @@ private static ShardBulkInferenceActionFilter createFilter( InferenceServiceRegistry inferenceServiceRegistry = mock(InferenceServiceRegistry.class); when(inferenceServiceRegistry.getService(any())).thenReturn(Optional.of(inferenceService)); + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.isAllowed(InferencePlugin.INFERENCE_API_FEATURE)).thenReturn(isLicenseValidForInference); + return new ShardBulkInferenceActionFilter( createClusterService(useLegacyFormat), inferenceServiceRegistry, modelRegistry, + licenseState, batchSize ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/InferenceServiceNodeLocalRateLimitCalculatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/InferenceServiceNodeLocalRateLimitCalculatorTests.java index 05ee936c23fd7..569c380953816 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/InferenceServiceNodeLocalRateLimitCalculatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/InferenceServiceNodeLocalRateLimitCalculatorTests.java @@ -10,55 +10,60 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculator.DEFAULT_MAX_NODES_PER_GROUPING; import static org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculator.SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0) public class InferenceServiceNodeLocalRateLimitCalculatorTests extends ESIntegTestCase { + private static final Integer RATE_LIMIT_ASSIGNMENT_MAX_WAIT_TIME_IN_SECONDS = 15; + public void setUp() throws Exception { super.setUp(); + assumeTrue( + "If inference_cluster_aware_rate_limiting_feature_flag_enabled=false we'll fallback to " + + "NoopNodeLocalRateLimitCalculator, which shouldn't be tested by this class.", + InferenceAPIClusterAwareRateLimitingFeature.INFERENCE_API_CLUSTER_AWARE_RATE_LIMITING_FEATURE_FLAG.isEnabled() + ); } - public void testInitialClusterGrouping_Correct() { + public void testInitialClusterGrouping_Correct() throws Exception { // Start with 2-5 nodes var numNodes = randomIntBetween(2, 5); var nodeNames = internalCluster().startNodes(numNodes); ensureStableCluster(numNodes); - RateLimitAssignment firstAssignment = null; + var firstCalculator = getCalculatorInstance(internalCluster(), nodeNames.get(0)); + waitForRateLimitingAssignments(firstCalculator); - for (String nodeName : nodeNames) { - var calculator = internalCluster().getInstance(InferenceServiceNodeLocalRateLimitCalculator.class, nodeName); - - // Check first node's assignments - if (firstAssignment == null) { - // Get assignment for a specific service (e.g., EIS) - firstAssignment = calculator.getRateLimitAssignment(ElasticInferenceService.NAME, TaskType.SPARSE_EMBEDDING); - - assertNotNull(firstAssignment); - // Verify there are assignments for this service - assertFalse(firstAssignment.responsibleNodes().isEmpty()); - } else { - // Verify other nodes see the same assignment - var currentAssignment = calculator.getRateLimitAssignment(ElasticInferenceService.NAME, TaskType.SPARSE_EMBEDDING); - assertEquals(firstAssignment, currentAssignment); - } + RateLimitAssignment firstAssignment = firstCalculator.getRateLimitAssignment( + ElasticInferenceService.NAME, + TaskType.SPARSE_EMBEDDING + ); + + // Verify that all other nodes land on the same assignment + for (String nodeName : nodeNames.subList(1, nodeNames.size())) { + var calculator = getCalculatorInstance(internalCluster(), nodeName); + waitForRateLimitingAssignments(calculator); + var currentAssignment = calculator.getRateLimitAssignment(ElasticInferenceService.NAME, TaskType.SPARSE_EMBEDDING); + assertEquals(firstAssignment, currentAssignment); } } - public void testNumberOfNodesPerGroup_Decreases_When_NodeLeavesCluster() throws IOException { + public void testNumberOfNodesPerGroup_Decreases_When_NodeLeavesCluster() throws Exception { // Start with 3-5 nodes var numNodes = randomIntBetween(3, 5); var nodeNames = internalCluster().startNodes(numNodes); @@ -77,7 +82,8 @@ public void testNumberOfNodesPerGroup_Decreases_When_NodeLeavesCluster() throws ensureStableCluster(currentNumberOfNodes); } - var calculator = internalCluster().getInstance(InferenceServiceNodeLocalRateLimitCalculator.class, nodeLeftInCluster); + var calculator = getCalculatorInstance(internalCluster(), nodeLeftInCluster); + waitForRateLimitingAssignments(calculator); Set supportedServices = SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.keySet(); @@ -93,13 +99,14 @@ public void testNumberOfNodesPerGroup_Decreases_When_NodeLeavesCluster() throws } } - public void testGrouping_RespectsMaxNodesPerGroupingLimit() { + public void testGrouping_RespectsMaxNodesPerGroupingLimit() throws Exception { // Start with more nodes possible per grouping var numNodes = DEFAULT_MAX_NODES_PER_GROUPING + randomIntBetween(1, 3); var nodeNames = internalCluster().startNodes(numNodes); ensureStableCluster(numNodes); - var calculator = internalCluster().getInstance(InferenceServiceNodeLocalRateLimitCalculator.class, nodeNames.get(0)); + var calculator = getCalculatorInstance(internalCluster(), nodeNames.get(0)); + waitForRateLimitingAssignments(calculator); Set supportedServices = SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.keySet(); @@ -111,13 +118,14 @@ public void testGrouping_RespectsMaxNodesPerGroupingLimit() { } } - public void testInitialRateLimitsCalculation_Correct() throws IOException { + public void testInitialRateLimitsCalculation_Correct() throws Exception { // Start with max nodes per grouping (=3) int numNodes = DEFAULT_MAX_NODES_PER_GROUPING; var nodeNames = internalCluster().startNodes(numNodes); ensureStableCluster(numNodes); - var calculator = internalCluster().getInstance(InferenceServiceNodeLocalRateLimitCalculator.class, nodeNames.get(0)); + var calculator = getCalculatorInstance(internalCluster(), nodeNames.get(0)); + waitForRateLimitingAssignments(calculator); Set supportedServices = SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.keySet(); @@ -129,7 +137,7 @@ public void testInitialRateLimitsCalculation_Correct() throws IOException { if ((service instanceof SenderService senderService)) { var sender = senderService.getSender(); - if (sender instanceof HttpRequestSender httpSender) { + if (sender instanceof HttpRequestSender) { var assignment = calculator.getRateLimitAssignment(service.name(), TaskType.SPARSE_EMBEDDING); assertNotNull(assignment); @@ -141,13 +149,14 @@ public void testInitialRateLimitsCalculation_Correct() throws IOException { } } - public void testRateLimits_Decrease_OnNodeJoin() { + public void testRateLimits_Decrease_OnNodeJoin() throws Exception { // Start with 2 nodes var initialNodes = 2; var nodeNames = internalCluster().startNodes(initialNodes); ensureStableCluster(initialNodes); - var calculator = internalCluster().getInstance(InferenceServiceNodeLocalRateLimitCalculator.class, nodeNames.get(0)); + var calculator = getCalculatorInstance(internalCluster(), nodeNames.get(0)); + waitForRateLimitingAssignments(calculator); for (var serviceName : SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.keySet()) { var configs = SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.get(serviceName); @@ -159,6 +168,7 @@ public void testRateLimits_Decrease_OnNodeJoin() { // Add a new node internalCluster().startNode(); ensureStableCluster(initialNodes + 1); + waitForRateLimitingAssignments(calculator); // Get updated assignments var updatedAssignment = calculator.getRateLimitAssignment(serviceName, config.taskType()); @@ -169,13 +179,14 @@ public void testRateLimits_Decrease_OnNodeJoin() { } } - public void testRateLimits_Increase_OnNodeLeave() throws IOException { + public void testRateLimits_Increase_OnNodeLeave() throws Exception { // Start with max nodes per grouping (=3) int numNodes = DEFAULT_MAX_NODES_PER_GROUPING; var nodeNames = internalCluster().startNodes(numNodes); ensureStableCluster(numNodes); - var calculator = internalCluster().getInstance(InferenceServiceNodeLocalRateLimitCalculator.class, nodeNames.get(0)); + var calculator = getCalculatorInstance(internalCluster(), nodeNames.get(0)); + waitForRateLimitingAssignments(calculator); for (var serviceName : SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.keySet()) { var configs = SERVICE_NODE_LOCAL_RATE_LIMIT_CONFIGS.get(serviceName); @@ -188,6 +199,7 @@ public void testRateLimits_Increase_OnNodeLeave() throws IOException { var nodeToRemove = nodeNames.get(numNodes - 1); internalCluster().stopNode(nodeToRemove); ensureStableCluster(numNodes - 1); + waitForRateLimitingAssignments(calculator); // Get updated assignments var updatedAssignment = calculator.getRateLimitAssignment(serviceName, config.taskType()); @@ -202,4 +214,33 @@ public void testRateLimits_Increase_OnNodeLeave() throws IOException { protected Collection> nodePlugins() { return Arrays.asList(LocalStateInferencePlugin.class); } + + private InferenceServiceNodeLocalRateLimitCalculator getCalculatorInstance(InternalTestCluster internalTestCluster, String nodeName) { + InferenceServiceRateLimitCalculator calculatorInstance = internalTestCluster.getInstance( + InferenceServiceRateLimitCalculator.class, + nodeName + ); + assertThat( + "[" + + InferenceServiceNodeLocalRateLimitCalculatorTests.class.getName() + + "] should use [" + + InferenceServiceNodeLocalRateLimitCalculator.class.getName() + + "] as implementation for [" + + InferenceServiceRateLimitCalculator.class.getName() + + "]. Provided implementation was [" + + calculatorInstance.getClass().getName() + + "].", + calculatorInstance, + instanceOf(InferenceServiceNodeLocalRateLimitCalculator.class) + ); + return (InferenceServiceNodeLocalRateLimitCalculator) calculatorInstance; + } + + private void waitForRateLimitingAssignments(InferenceServiceNodeLocalRateLimitCalculator calculator) throws Exception { + assertBusy(() -> { + var assignment = calculator.getRateLimitAssignment(ElasticInferenceService.NAME, TaskType.SPARSE_EMBEDDING); + assertNotNull(assignment); + assertFalse(assignment.responsibleNodes().isEmpty()); + }, RATE_LIMIT_ASSIGNMENT_MAX_WAIT_TIME_IN_SECONDS, TimeUnit.SECONDS); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java index 3b95c4ba86e59..4d88eabb105db 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java @@ -34,6 +34,7 @@ public class SenderExecutableActionTests extends ESTestCase { private static final String failedToSendRequestErrorMessage = "test failed"; + private static final String failureExceptionMessage = failedToSendRequestErrorMessage + ". Cause: test"; private Sender sender; private RequestManager requestManager; private SenderExecutableAction executableAction; @@ -96,7 +97,7 @@ public void testSendThrowingExceptionIsWrapped() { execute(actualException); assertThat(actualException.get(), notNullValue()); - assertThat(actualException.get().getMessage(), is(failedToSendRequestErrorMessage)); + assertThat(actualException.get().getMessage(), is(failureExceptionMessage)); assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); assertThat(actualException.get().getCause(), sameInstance(expectedException)); } @@ -110,7 +111,7 @@ public void testSenderReturnedExceptionIsWrapped() { execute(actualException); assertThat(actualException.get(), notNullValue()); - assertThat(actualException.get().getMessage(), is(failedToSendRequestErrorMessage)); + assertThat(actualException.get().getMessage(), is(failureExceptionMessage)); assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); assertThat(actualException.get().getCause(), sameInstance(expectedException)); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionActionTests.java new file mode 100644 index 0000000000000..e6f9940350bee --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchCompletionActionTests.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.alibabacloudsearch; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.ServiceComponentsTests; +import org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionModelTests; +import org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionTaskSettingsTests; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.lucene.tests.util.LuceneTestCase.expectThrows; +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; +import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion; +import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +public class AlibabaCloudSearchCompletionActionTests extends ESTestCase { + + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws IOException { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testExecute_Success() { + var sender = mock(Sender.class); + + var resultString = randomAlphaOfLength(100); + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(3); + listener.onResponse(new ChatCompletionResults(List.of(new ChatCompletionResults.Result(resultString)))); + + return Void.TYPE; + }).when(sender).send(any(), any(), any(), any()); + var action = createAction(threadPool, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new ChatCompletionInput(List.of(randomAlphaOfLength(10))), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + assertThat(result.asMap(), is(buildExpectationCompletion(List.of(resultString)))); + } + + public void testExecute_ListenerThrowsElasticsearchException_WhenSenderThrowsElasticsearchException() { + var sender = mock(Sender.class); + doThrow(new ElasticsearchException("error")).when(sender).send(any(), any(), any(), any()); + var action = createAction(threadPool, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new ChatCompletionInput(List.of(randomAlphaOfLength(10))), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + assertThat(thrownException.getMessage(), is("error")); + } + + public void testExecute_ListenerThrowsInternalServerError_WhenSenderThrowsException() { + var sender = mock(Sender.class); + doThrow(new RuntimeException("error")).when(sender).send(any(), any(), any(), any()); + var action = createAction(threadPool, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new ChatCompletionInput(List.of(randomAlphaOfLength(10))), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + assertThat(thrownException.getMessage(), is(constructFailedToSendRequestMessage("AlibabaCloud Search completion"))); + } + + public void testExecute_ThrowsIllegalArgumentException_WhenInputIsNotChatCompletionInput() { + var action = createAction(threadPool, mock(Sender.class)); + + PlainActionFuture listener = new PlainActionFuture<>(); + assertThrows(IllegalArgumentException.class, () -> { + action.execute(new DocumentsOnlyInput(List.of(randomAlphaOfLength(10))), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + }); + } + + public void testExecute_ListenerThrowsElasticsearchStatusException_WhenInputSizeIsEven() { + var action = createAction(threadPool, mock(Sender.class)); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute( + new ChatCompletionInput(List.of(randomAlphaOfLength(10), randomAlphaOfLength(10))), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + thrownException.getMessage(), + is( + "Alibaba Completion's inputs must be an odd number. The last input is the current query, " + + "all preceding inputs are the completion history as pairs of user input and the assistant's response." + ) + ); + assertThat(thrownException.status(), is(RestStatus.BAD_REQUEST)); + } + + private ExecutableAction createAction(ThreadPool threadPool, Sender sender) { + var model = AlibabaCloudSearchCompletionModelTests.createModel( + "completion_test", + TaskType.COMPLETION, + AlibabaCloudSearchCompletionServiceSettingsTests.getServiceSettingsMap("completion_test", "host", "default"), + AlibabaCloudSearchCompletionTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var serviceComponents = ServiceComponentsTests.createWithEmptySettings(threadPool); + return new AlibabaCloudSearchCompletionAction(sender, model, serviceComponents); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java index f0de37ceaaf98..b9f0281c20c82 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; @@ -171,15 +170,13 @@ public void testCreate_ChatCompletionModel_FailsFromInvalidResponseFormat() thro PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + var failureCauseMessage = "Failed to find required field [content] in Anthropic chat completions response"; var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), - is(format("Failed to send Anthropic chat completions request to [%s]", getUrl(webServer))) - ); - assertThat( - thrownException.getCause().getMessage(), - is("Failed to find required field [content] in Anthropic chat completions response") + is("Failed to send Anthropic chat completions request. Cause: " + failureCauseMessage) ); + assertThat(thrownException.getCause().getMessage(), is(failureCauseMessage)); assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java index 2065a726b7589..fb538cfc5d55b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java @@ -42,7 +42,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -160,8 +159,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -174,10 +172,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Anthropic chat completions request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Anthropic chat completions request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -191,10 +186,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Anthropic chat completions request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Anthropic chat completions request. Cause: failed")); } public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException { @@ -241,7 +233,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc private ExecutableAction createAction(String url, String apiKey, String modelName, int maxTokens, Sender sender) { var model = AnthropicChatCompletionModelTests.createChatCompletionModel(url, apiKey, modelName, maxTokens); var requestCreator = AnthropicCompletionRequestManager.of(model, threadPool); - var errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Anthropic chat completions"); + var errorMessage = constructFailedToSendRequestMessage("Anthropic chat completions"); return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, "Anthropic chat completions"); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java index 7e1e3e55caed8..df124c35a4689 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java @@ -219,12 +219,13 @@ public void testCreate_AzureOpenAiEmbeddingsModel_FailsFromInvalidResponseFormat PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + var failureCauseMessage = "Failed to find required field [data] in OpenAI embeddings response"; var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), - is(format("Failed to send Azure OpenAI embeddings request to [%s]", getUrl(webServer))) + is(format("Failed to send Azure OpenAI embeddings request. Cause: %s", failureCauseMessage)) ); - assertThat(thrownException.getCause().getMessage(), is("Failed to find required field [data] in OpenAI embeddings response")); + assertThat(thrownException.getCause().getMessage(), is(failureCauseMessage)); assertThat(webServer.requests(), hasSize(1)); validateRequestWithApiKey(webServer.requests().get(0), "apikey"); @@ -592,10 +593,11 @@ public void testInfer_AzureOpenAiCompletionModel_FailsFromInvalidResponseFormat( PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new ChatCompletionInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + var failureCauseMessage = "Failed to find required field [choices] in Azure OpenAI completions response"; var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), - is(format("Failed to send Azure OpenAI completion request to [%s]", getUrl(webServer))) + is(format("Failed to send Azure OpenAI completion request. Cause: %s", failureCauseMessage)) ); assertThat( thrownException.getCause().getMessage(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java index dca12dfda9c98..f2885373a3b32 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -153,8 +152,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -167,7 +165,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI completion request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Azure OpenAI completion request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -181,7 +179,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI completion request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Azure OpenAI completion request. Cause: failed")); } private ExecutableAction createAction( @@ -197,7 +195,7 @@ private ExecutableAction createAction( var model = createCompletionModel(resourceName, deploymentId, apiVersion, user, apiKey, null, inferenceEntityId); model.setUri(new URI(getUrl(webServer))); var requestCreator = new AzureOpenAiCompletionRequestManager(model, threadPool); - var errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI completion"); + var errorMessage = constructFailedToSendRequestMessage("Azure OpenAI completion"); return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, "Azure OpenAI completion"); } catch (URISyntaxException e) { throw new RuntimeException(e); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java index 4c07ce81eb4cc..33d0e7d02a380 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java @@ -42,7 +42,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -149,8 +148,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -163,15 +161,14 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Azure OpenAI embeddings request. Cause: failed")); } public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled_WhenUrlIsNull() { var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -184,7 +181,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Azure OpenAI embeddings request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -198,7 +195,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Azure OpenAI embeddings request. Cause: failed")); } private ExecutableAction createAction( @@ -215,7 +212,7 @@ private ExecutableAction createAction( model = createModel(resourceName, deploymentId, apiVersion, user, apiKey, null, inferenceEntityId); model.setUri(new URI(getUrl(webServer))); var requestCreator = new AzureOpenAiEmbeddingsRequestManager(model, TruncatorTests.createTruncator(), threadPool); - var errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI embeddings"); + var errorMessage = constructFailedToSendRequestMessage("Azure OpenAI embeddings"); return new SenderExecutableAction(sender, requestCreator, errorMessage); } catch (URISyntaxException e) { throw new RuntimeException(e); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java index c5871adb34864..6e98389728f13 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -225,8 +224,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -239,15 +237,14 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Cohere completion request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Cohere completion request. Cause: failed")); } public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled_WhenUrlIsNull() { var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -260,7 +257,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Cohere completion request", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Cohere completion request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -274,7 +271,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Cohere completion request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Cohere completion request. Cause: failed")); } public void testExecute_ThrowsExceptionWithNullUrl() { @@ -288,7 +285,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("Failed to send Cohere completion request")); + assertThat(thrownException.getMessage(), is("Failed to send Cohere completion request. Cause: failed")); } public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException { @@ -346,7 +343,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc private ExecutableAction createAction(String url, String apiKey, @Nullable String modelName, Sender sender) { var model = CohereCompletionModelTests.createModel(url, apiKey, modelName); var requestManager = CohereCompletionRequestManager.of(model, threadPool); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "Cohere completion"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Cohere completion"); return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, "Cohere completion"); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java index fe0eb782eddfc..5bd919e7bbc17 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java @@ -45,7 +45,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -272,8 +271,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -286,18 +284,14 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - MatcherAssert.assertThat( - thrownException.getMessage(), - is(format("Failed to send Cohere embeddings request to [%s]", getUrl(webServer))) - ); + MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request. Cause: failed")); } public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled_WhenUrlIsNull() { var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -310,7 +304,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request")); + MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -324,10 +318,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - MatcherAssert.assertThat( - thrownException.getMessage(), - is(format("Failed to send Cohere embeddings request to [%s]", getUrl(webServer))) - ); + MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request. Cause: failed")); } public void testExecute_ThrowsExceptionWithNullUrl() { @@ -341,7 +332,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request")); + MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request. Cause: failed")); } private ExecutableAction createAction( @@ -353,10 +344,7 @@ private ExecutableAction createAction( Sender sender ) { var model = CohereEmbeddingsModelTests.createModel(url, apiKey, taskSettings, 1024, 1024, modelName, embeddingType); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - model.getServiceSettings().getCommonSettings().uri(), - "Cohere embeddings" - ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Cohere embeddings"); var requestCreator = CohereEmbeddingsRequestManager.of(model, threadPool); return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java index e1d2ee56733e3..28e182aa2d435 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java @@ -124,7 +124,7 @@ public void testExecute_ReturnsSuccessfulResponse_ForElserAction() throws IOExce assertThat(requestMap.get("input"), instanceOf(List.class)); var inputList = (List) requestMap.get("input"); assertThat(inputList, contains("hello world")); - assertThat(requestMap.get("model_id"), is("my-model-id")); + assertThat(requestMap.get("model"), is("my-model-id")); } } @@ -179,7 +179,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOEx assertThat(requestMap.get("input"), instanceOf(List.class)); var inputList = (List) requestMap.get("input"); assertThat(inputList, contains("hello world")); - assertThat(requestMap.get("model_id"), is("my-model-id")); + assertThat(requestMap.get("model"), is("my-model-id")); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java index ff17bbf66e02a..0e0c9f8b8a0e3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java @@ -39,7 +39,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -170,8 +169,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -184,10 +182,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Google AI Studio completion request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Google AI Studio completion request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -201,10 +196,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Google AI Studio completion request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Google AI Studio completion request. Cause: failed")); } public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException { @@ -272,7 +264,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc private ExecutableAction createAction(String url, String apiKey, String modelName, Sender sender) { var model = GoogleAiStudioCompletionModelTests.createModel(modelName, url, apiKey); var requestManager = new GoogleAiStudioCompletionRequestManager(model, threadPool); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(false), "Google AI Studio completion"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google AI Studio completion"); return new SingleInputSenderExecutableAction( sender, requestManager, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java index 27862f7309877..1cdce75d3ae0c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java @@ -38,7 +38,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -150,8 +149,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -164,10 +162,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Google AI Studio embeddings request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Google AI Studio embeddings request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -181,16 +176,13 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Google AI Studio embeddings request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Google AI Studio embeddings request. Cause: failed")); } private ExecutableAction createAction(String url, String apiKey, String modelName, Sender sender) { var model = createModel(modelName, apiKey, url); var requestManager = new GoogleAiStudioEmbeddingsRequestManager(model, TruncatorTests.createTruncator(), threadPool); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google AI Studio embeddings"); return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java index edfc447d5337e..74d19803a5557 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -85,8 +84,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -99,10 +97,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Google Vertex AI embeddings request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Google Vertex AI embeddings request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -116,16 +111,13 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send Google Vertex AI embeddings request to [%s]", getUrl(webServer))) - ); + assertThat(thrownException.getMessage(), is("Failed to send Google Vertex AI embeddings request. Cause: failed")); } private ExecutableAction createAction(String url, String location, String projectId, String modelName, Sender sender) { var model = createModel(location, projectId, modelName, url, "{}"); var requestManager = new GoogleVertexAiEmbeddingsRequestManager(model, TruncatorTests.createTruncator(), threadPool); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI embeddings"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google Vertex AI embeddings"); return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java index 491e17fc8c0a3..14704b3741eb8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -83,8 +82,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -97,7 +95,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Google Vertex AI rerank request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Google Vertex AI rerank request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -111,12 +109,12 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send Google Vertex AI rerank request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send Google Vertex AI rerank request. Cause: failed")); } private ExecutableAction createAction(String url, String projectId, Sender sender) { var model = GoogleVertexAiRerankModelTests.createModel(url, projectId, null); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI rerank"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("Google Vertex AI rerank"); var requestManager = GoogleVertexAiRerankRequestManager.of(model, threadPool); return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java index 848ca790d677d..3ea49cbfabe59 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java @@ -72,8 +72,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -88,7 +87,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled assertThat( thrownException.getMessage(), - is(format("Failed to send Hugging Face test action request from inference entity id [%s]", "inferenceEntityId")) + is(format("Failed to send Hugging Face test action request from inference entity id [%s]. Cause: failed", "inferenceEntityId")) ); } @@ -105,7 +104,7 @@ public void testExecute_ThrowsException() { assertThat( thrownException.getMessage(), - is(format("Failed to send Hugging Face test action request from inference entity id [%s]", "inferenceEntityId")) + is(format("Failed to send Hugging Face test action request from inference entity id [%s]. Cause: failed", "inferenceEntityId")) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxEmbeddingsActionTests.java index f43bb1e10e08b..7ac400667a819 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/ibmwatsonx/IbmWatsonxEmbeddingsActionTests.java @@ -43,7 +43,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -155,8 +154,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var apiVersion = "apiVersion"; doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -169,7 +167,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send IBM Watsonx embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send IBM Watsonx embeddings request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -189,7 +187,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send IBM Watsonx embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send IBM Watsonx embeddings request. Cause: failed")); } private ExecutableAction createAction( @@ -203,7 +201,7 @@ private ExecutableAction createAction( ) { var model = createModel(modelName, projectId, uri, apiVersion, apiKey, url); var requestManager = new IbmWatsonxEmbeddingsRequestManagerWithoutAuth(model, TruncatorTests.createTruncator(), threadPool); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "IBM Watsonx embeddings"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage("IBM Watsonx embeddings"); return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java index 8ea7e6c2bdb8d..2b2c9d0c3260d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java @@ -273,9 +273,13 @@ public void testCreate_OpenAiEmbeddingsModel_FailsFromInvalidResponseFormat() th PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + var failureCauseMessage = "Failed to find required field [data] in OpenAI embeddings response"; var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send OpenAI embeddings request to [%s]", getUrl(webServer)))); - assertThat(thrownException.getCause().getMessage(), is("Failed to find required field [data] in OpenAI embeddings response")); + assertThat( + thrownException.getMessage(), + is(format("Failed to send OpenAI embeddings request. Cause: %s", failureCauseMessage)) + ); + assertThat(thrownException.getCause().getMessage(), is(failureCauseMessage)); assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); @@ -529,15 +533,13 @@ public void testCreate_OpenAiChatCompletionModel_FailsFromInvalidResponseFormat( PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + var failureCauseMessage = "Failed to find required field [choices] in OpenAI chat completions response"; var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), - is(format("Failed to send OpenAI chat completions request to [%s]", getUrl(webServer))) - ); - assertThat( - thrownException.getCause().getMessage(), - is("Failed to find required field [choices] in OpenAI chat completions response") + is(format("Failed to send OpenAI chat completions request. Cause: %s", failureCauseMessage)) ); + assertThat(thrownException.getCause().getMessage(), is(failureCauseMessage)); assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java index e248f77fe7728..c96372eadfbc2 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -171,8 +170,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -185,15 +183,14 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send OpenAI chat completions request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI chat completions request. Cause: failed")); } public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled_WhenUrlIsNull() { var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -206,7 +203,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("Failed to send OpenAI chat completions request")); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI chat completions request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -220,7 +217,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send OpenAI chat completions request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI chat completions request. Cause: failed")); } public void testExecute_ThrowsExceptionWithNullUrl() { @@ -234,7 +231,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("Failed to send OpenAI chat completions request")); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI chat completions request. Cause: failed")); } public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException { @@ -286,7 +283,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc private ExecutableAction createAction(String url, String org, String apiKey, String modelName, @Nullable String user, Sender sender) { var model = createCompletionModel(url, org, apiKey, modelName, user); var requestCreator = OpenAiCompletionRequestManager.of(model, threadPool); - var errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI chat completions"); + var errorMessage = constructFailedToSendRequestMessage("OpenAI chat completions"); return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, "OpenAI chat completions"); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java index 509dd144a1d1f..c8a0e1c398d4b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; @@ -158,8 +157,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -172,15 +170,14 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send OpenAI embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request. Cause: failed")); } public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled_WhenUrlIsNull() { var sender = mock(Sender.class); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = invocation.getArgument(3); listener.onFailure(new IllegalStateException("failed")); return Void.TYPE; @@ -193,7 +190,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request")); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request. Cause: failed")); } public void testExecute_ThrowsException() { @@ -207,7 +204,7 @@ public void testExecute_ThrowsException() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is(format("Failed to send OpenAI embeddings request to [%s]", getUrl(webServer)))); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request. Cause: failed")); } public void testExecute_ThrowsExceptionWithNullUrl() { @@ -221,13 +218,13 @@ public void testExecute_ThrowsExceptionWithNullUrl() { var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request")); + assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request. Cause: failed")); } private ExecutableAction createAction(String url, String org, String apiKey, String modelName, @Nullable String user, Sender sender) { var model = createModel(url, org, apiKey, modelName, user); var requestCreator = OpenAiEmbeddingsRequestManager.of(model, TruncatorTests.createTruncator(), threadPool); - var errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI embeddings"); + var errorMessage = constructFailedToSendRequestMessage("OpenAI embeddings"); return new SenderExecutableAction(sender, requestCreator, errorMessage); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java index aa27bf0d2fc81..fc293b5f55668 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java @@ -41,7 +41,6 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.concurrent.CancellationException; -import java.util.concurrent.Flow; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -174,7 +173,7 @@ public void testStream_FailedCallsOnFailure() throws Exception { try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool, mockThrottlerManager())) { client.start(); - PlainActionFuture> listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); client.stream(httpPost, HttpClientContext.create(), listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -196,7 +195,7 @@ public void testStream_CancelledCallsOnFailure() throws Exception { try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool, mockThrottlerManager())) { client.start(); - PlainActionFuture> listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); client.stream(httpPost, HttpClientContext.create(), listener); var thrownException = expectThrows(CancellationException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java index a400b67b3761f..672dd05abc91d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java @@ -14,8 +14,10 @@ import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -53,7 +55,7 @@ public class StreamingHttpResultPublisherTests extends ESTestCase { private static final long maxBytes = message.length; private ThreadPool threadPool; private HttpSettings settings; - private ActionListener> listener; + private final AtomicReference> result = new AtomicReference<>(null); private StreamingHttpResultPublisher publisher; @Before @@ -61,12 +63,21 @@ public void setUp() throws Exception { super.setUp(); threadPool = mock(ThreadPool.class); settings = mock(HttpSettings.class); - listener = spy(ActionListener.noop()); when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(settings.getMaxResponseSize()).thenReturn(ByteSizeValue.ofBytes(maxBytes)); - publisher = new StreamingHttpResultPublisher(threadPool, settings, listener); + publisher = new StreamingHttpResultPublisher(threadPool, settings, listener()); + } + + private ActionListener listener() { + return ActionListener.wrap(r -> result.set(Tuple.tuple(r, null)), e -> result.set(Tuple.tuple(null, e))); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + result.set(null); } /** @@ -76,7 +87,7 @@ public void setUp() throws Exception { */ public void testFirstResponseCallsListener() throws IOException { var latch = new CountDownLatch(1); - var listener = ActionTestUtils.>assertNoFailureListener(r -> latch.countDown()); + var listener = ActionTestUtils.assertNoFailureListener(r -> latch.countDown()); publisher = new StreamingHttpResultPublisher(threadPool, settings, listener); publisher.responseReceived(mock(HttpResponse.class)); @@ -92,7 +103,7 @@ public void testFirstResponseCallsListener() throws IOException { */ public void testNonEmptyFirstResponseCallsListener() throws IOException { var latch = new CountDownLatch(1); - var listener = ActionTestUtils.>assertNoFailureListener(r -> latch.countDown()); + var listener = ActionTestUtils.assertNoFailureListener(r -> latch.countDown()); publisher = new StreamingHttpResultPublisher(threadPool, settings, listener); when(settings.getMaxResponseSize()).thenReturn(ByteSizeValue.ofBytes(9000)); @@ -127,7 +138,7 @@ public void testSubscriberAndPublisherExchange() throws IOException { publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); // subscribe - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); assertThat("subscribe must call onSubscribe", subscriber.subscription, notNullValue()); assertThat("onNext should only be called once we have requested data", subscriber.httpResult, nullValue()); @@ -142,7 +153,7 @@ public void testSubscriberAndPublisherExchange() throws IOException { // publisher sends data publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); - assertThat("onNext was called with " + new String(message, StandardCharsets.UTF_8), subscriber.httpResult.body(), equalTo(message)); + assertThat("onNext was called with " + new String(message, StandardCharsets.UTF_8), subscriber.httpResult, equalTo(message)); } /** @@ -157,7 +168,7 @@ public void testNon200Response() throws IOException { publisher.close(); // subscriber requests data - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); assertThat("subscribe must call onSubscribe", subscriber.subscription, notNullValue()); subscriber.requestData(); assertThat("onNext was called with the initial HttpResponse", subscriber.httpResult, notNullValue()); @@ -187,7 +198,7 @@ public void testPauseApache() throws IOException { public void testResumeApache() throws IOException { var subscriber = new TestSubscriber(); publisher.responseReceived(mock(HttpResponse.class)); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); subscriber.requestData(); subscriber.httpResult = null; @@ -212,7 +223,7 @@ public void testTotalBytesDecrement() throws IOException { var subscriber = new TestSubscriber(); publisher.responseReceived(mock(HttpResponse.class)); publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); subscriber.requestData(); subscriber.httpResult = null; @@ -243,7 +254,7 @@ public void testTotalBytesDecrement() throws IOException { public void testErrorBeforeRequest() { var exception = new NullPointerException("test"); publisher.failed(exception); - verify(listener).onFailure(exception); + assertThat(result.get().v2(), equalTo(exception)); } /** @@ -361,21 +372,17 @@ public void testCancelAfterRequest() { * When cancel is called * Then we only send onComplete once */ - public void testCancelIsIdempotent() throws IOException { - Flow.Subscriber subscriber = mock(); + public void testCancelIsIdempotent() { + Flow.Subscriber subscriber = mock(); + + publisher.responseReceived(mock()); var subscription = ArgumentCaptor.forClass(Flow.Subscription.class); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); verify(subscriber).onSubscribe(subscription.capture()); - publisher.responseReceived(mock()); - publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); - subscription.getValue().request(1); - - subscription.getValue().request(1); + subscription.getValue().request(2); publisher.cancel(); - verify(subscriber, times(1)).onComplete(); - subscription.getValue().request(1); publisher.cancel(); verify(subscriber, times(1)).onComplete(); } @@ -384,21 +391,17 @@ public void testCancelIsIdempotent() throws IOException { * When close is called * Then we only send onComplete once */ - public void testCloseIsIdempotent() throws IOException { - Flow.Subscriber subscriber = mock(); + public void testCloseIsIdempotent() { + Flow.Subscriber subscriber = mock(); + + publisher.responseReceived(mock()); var subscription = ArgumentCaptor.forClass(Flow.Subscription.class); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); verify(subscriber).onSubscribe(subscription.capture()); - publisher.responseReceived(mock()); - publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); - subscription.getValue().request(1); - - subscription.getValue().request(1); + subscription.getValue().request(2); publisher.close(); - verify(subscriber, times(1)).onComplete(); - subscription.getValue().request(1); publisher.close(); verify(subscriber, times(1)).onComplete(); } @@ -409,20 +412,16 @@ public void testCloseIsIdempotent() throws IOException { */ public void testFailedIsIdempotent() throws IOException { var expectedException = new IllegalStateException("wow"); - Flow.Subscriber subscriber = mock(); + Flow.Subscriber subscriber = mock(); + + publisher.responseReceived(mock()); var subscription = ArgumentCaptor.forClass(Flow.Subscription.class); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); verify(subscriber).onSubscribe(subscription.capture()); - publisher.responseReceived(mock()); - publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); - subscription.getValue().request(1); - - subscription.getValue().request(1); + subscription.getValue().request(2); publisher.failed(expectedException); - verify(subscriber, times(1)).onError(eq(expectedException)); - subscription.getValue().request(1); publisher.failed(expectedException); verify(subscriber, times(1)).onError(eq(expectedException)); } @@ -492,10 +491,11 @@ public void testRequestingZeroFails() { * Then that subscriber should receive an IllegalStateException */ public void testDoubleSubscribeFails() { - publisher.subscribe(mock()); + publisher.responseReceived(mock()); + testPublisher().subscribe(mock()); var subscriber = new TestSubscriber(); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); assertThat(subscriber.throwable, notNullValue()); assertThat(subscriber.throwable, instanceOf(IllegalStateException.class)); } @@ -508,10 +508,10 @@ public void testDoubleSubscribeFails() { public void testReuseMlThread() throws ExecutionException, InterruptedException, TimeoutException { try { threadPool = spy(createThreadPool(inferenceUtilityPool())); - publisher = new StreamingHttpResultPublisher(threadPool, settings, listener); + publisher = new StreamingHttpResultPublisher(threadPool, settings, listener()); var subscriber = new TestSubscriber(); publisher.responseReceived(mock(HttpResponse.class)); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); CompletableFuture.runAsync(() -> { try { @@ -523,7 +523,7 @@ public void testReuseMlThread() throws ExecutionException, InterruptedException, }, threadPool.executor(UTILITY_THREAD_POOL_NAME)).get(5, TimeUnit.SECONDS); verify(threadPool, times(1)).executor(UTILITY_THREAD_POOL_NAME); assertThat("onNext was called with the initial HttpResponse", subscriber.httpResult, notNullValue()); - assertFalse("Expected HttpResult to have data", subscriber.httpResult.isBodyEmpty()); + assertNotEquals("Expected HttpResult to have data", 0, subscriber.httpResult.length); } finally { terminate(threadPool); } @@ -548,11 +548,11 @@ public void testCancelBreaksInfiniteLoop() throws Exception { return executorServiceSpy; }).when(threadPool).executor(UTILITY_THREAD_POOL_NAME); - publisher = new StreamingHttpResultPublisher(threadPool, settings, listener); + publisher = new StreamingHttpResultPublisher(threadPool, settings, listener()); publisher.responseReceived(mock(HttpResponse.class)); publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); // create an infinitely running Subscriber - var subscriber = new Flow.Subscriber() { + var subscriber = new Flow.Subscriber() { Flow.Subscription subscription; boolean completed = false; @@ -563,7 +563,7 @@ public void onSubscribe(Flow.Subscription subscription) { } @Override - public void onNext(HttpResult item) { + public void onNext(byte[] item) { try { publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); } catch (IOException e) { @@ -582,7 +582,7 @@ public void onComplete() { completed = true; } }; - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); // verify the thread has started assertThat("Thread should have started on subscribe", futureHolder.get(), notNullValue()); @@ -604,7 +604,7 @@ public void testOnlyRunOneAtATime() throws IOException { // start with a message published publisher.responseReceived(mock(HttpResponse.class)); TestSubscriber subscriber = new TestSubscriber() { - public void onNext(HttpResult item) { + public void onNext(byte[] item) { try { // publish a second message publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); @@ -617,7 +617,7 @@ public void onNext(HttpResult item) { super.onNext(item); } }; - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); verify(threadPool, times(0)).executor(UTILITY_THREAD_POOL_NAME); subscriber.requestData(); @@ -646,8 +646,9 @@ public boolean isCompleted() { } private TestSubscriber subscribe() { + publisher.responseReceived(mock()); var subscriber = new TestSubscriber(); - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); return subscriber; } @@ -655,12 +656,12 @@ private TestSubscriber runBefore(Runnable runDuringOnNext) throws IOException { publisher.responseReceived(mock(HttpResponse.class)); publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); TestSubscriber subscriber = new TestSubscriber() { - public void onNext(HttpResult item) { + public void onNext(byte[] item) { runDuringOnNext.run(); super.onNext(item); } }; - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); return subscriber; } @@ -668,19 +669,23 @@ private TestSubscriber runAfter(Runnable runDuringOnNext) throws IOException { publisher.responseReceived(mock(HttpResponse.class)); publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); TestSubscriber subscriber = new TestSubscriber() { - public void onNext(HttpResult item) { + public void onNext(byte[] item) { runDuringOnNext.run(); super.requestData(); super.onNext(item); } }; - publisher.subscribe(subscriber); + testPublisher().subscribe(subscriber); return subscriber; } - private static class TestSubscriber implements Flow.Subscriber { + private Flow.Publisher testPublisher() { + return result.get().v1().body(); + } + + private static class TestSubscriber implements Flow.Subscriber { private Flow.Subscription subscription; - private HttpResult httpResult; + private byte[] httpResult; private Throwable throwable; private boolean completed; @@ -690,7 +695,7 @@ public void onSubscribe(Flow.Subscription subscription) { } @Override - public void onNext(HttpResult item) { + public void onNext(byte[] item) { this.httpResult = item; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java index 0100c2812cdc0..16285a6bd9b9e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java @@ -24,15 +24,18 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.xpack.inference.external.http.HttpClient; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.StreamingHttpResult; import org.elasticsearch.xpack.inference.external.request.HttpRequestTests; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.junit.Before; +import org.mockito.ArgumentMatchers; import org.mockito.stubbing.Answer; import java.io.IOException; import java.net.UnknownHostException; import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.createDefaultRetrySettings; import static org.hamcrest.Matchers.instanceOf; @@ -42,7 +45,6 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.only; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -459,12 +461,12 @@ public void testSend_ReturnsFailure_WhenHttpResultsListenerCallsOnFailure_WithNo verifyNoMoreInteractions(httpClient); } - public void testStream() throws IOException { + public void testStreamSuccess() throws IOException { var httpClient = mock(HttpClient.class); - Flow.Publisher publisher = mock(); + StreamingHttpResult streamingHttpResult = new StreamingHttpResult(mockHttpResponse(), randomPublisher()); doAnswer(ans -> { - ActionListener> listener = ans.getArgument(2); - listener.onResponse(publisher); + ActionListener listener = ans.getArgument(2); + listener.onResponse(streamingHttpResult); return null; }).when(httpClient).stream(any(), any(), any()); @@ -479,7 +481,28 @@ public void testStream() throws IOException { verify(httpClient, times(1)).stream(any(), any(), any()); verifyNoMoreInteractions(httpClient); - verify(publisher, only()).subscribe(any(StreamingResponseHandler.class)); + verify(responseHandler, times(1)).parseResult(any(), ArgumentMatchers.>any()); + } + + private Flow.Publisher randomPublisher() { + var calls = new AtomicInteger(randomIntBetween(1, 4)); + return subscriber -> { + subscriber.onSubscribe(new Flow.Subscription() { + @Override + public void request(long n) { + if (calls.getAndDecrement() > 0) { + subscriber.onNext(randomByteArrayOfLength(3)); + } else { + subscriber.onComplete(); + } + } + + @Override + public void cancel() { + + } + }); + }; } public void testStream_ResponseHandlerDoesNotHandleStreams() throws IOException { @@ -549,6 +572,44 @@ public void testSend_DoesNotRetryIndefinitely() throws IOException { } } + public void testStream_DoesNotRetryIndefinitely() throws IOException { + var threadPool = new TestThreadPool(getTestName()); + try { + var httpClient = mock(HttpClient.class); + doAnswer(ans -> { + ActionListener listener = ans.getArgument(2); + listener.onFailure(new ConnectionClosedException("failed")); + return null; + }).when(httpClient).stream(any(), any(), any()); + + var handler = mock(ResponseHandler.class); + when(handler.canHandleStreamingResponses()).thenReturn(true); + + var retrier = new RetryingHttpSender( + httpClient, + mock(ThrottlerManager.class), + createDefaultRetrySettings(), + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + + var listener = new PlainActionFuture(); + var request = mockRequest(); + when(request.isStreaming()).thenReturn(true); + retrier.send(mock(Logger.class), request, () -> false, handler, listener); + + // Assert that the retrying sender stopped after max retires even though the exception is retryable + var thrownException = expectThrows(UncategorizedExecutionException.class, () -> listener.actionGet(TIMEOUT)); + assertThat(thrownException.getCause(), instanceOf(ConnectionClosedException.class)); + assertThat(thrownException.getMessage(), is("Failed execution")); + assertThat(thrownException.getSuppressed().length, is(0)); + verify(httpClient, times(RetryingHttpSender.MAX_RETIES)).stream(any(), any(), any()); + verifyNoMoreInteractions(httpClient); + } finally { + terminate(threadPool); + } + } + public void testSend_DoesNotRetryIndefinitely_WithAlwaysRetryingResponseHandler() throws IOException { var threadPool = new TestThreadPool(getTestName()); try { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/StreamingResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/StreamingResponseHandlerTests.java deleted file mode 100644 index 6894c9a715f22..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/StreamingResponseHandlerTests.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.http.retry; - -import org.apache.http.HttpResponse; -import org.apache.http.StatusLine; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.request.Request; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; -import org.junit.After; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import java.util.concurrent.Flow; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class StreamingResponseHandlerTests extends ESTestCase { - @Mock - private HttpResponse response; - @Mock - private ThrottlerManager throttlerManager; - @Mock - private Logger logger; - @Mock - private Request request; - @Mock - private ResponseHandler responseHandler; - @Mock - private Flow.Subscriber downstreamSubscriber; - @InjectMocks - private StreamingResponseHandler streamingResponseHandler; - private AutoCloseable mocks; - private HttpResult item; - - @Before - public void setUp() throws Exception { - super.setUp(); - mocks = MockitoAnnotations.openMocks(this); - item = new HttpResult(response, new byte[0]); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - mocks.close(); - } - - public void testResponseHandlerFailureIsForwardedToSubscriber() { - var upstreamSubscription = upstreamSubscription(); - var expectedException = new RetryException(true, "ah"); - doThrow(expectedException).when(responseHandler).validateResponse(any(), any(), any(), any()); - - var statusLine = mock(StatusLine.class); - when(statusLine.getStatusCode()).thenReturn(404); - when(statusLine.getReasonPhrase()).thenReturn("not found"); - when(response.getStatusLine()).thenReturn(statusLine); - - streamingResponseHandler.onNext(item); - - verify(upstreamSubscription, times(1)).cancel(); - verify(downstreamSubscriber, times(1)).onError(expectedException); - } - - @SuppressWarnings("unchecked") - private Flow.Subscription upstreamSubscription() { - var upstreamSubscription = mock(Flow.Subscription.class); - streamingResponseHandler.onSubscribe(upstreamSubscription); - streamingResponseHandler.subscribe(downstreamSubscriber); - return upstreamSubscription; - } - - public void testOnNextCallsDownstream() { - upstreamSubscription(); - - streamingResponseHandler.onNext(item); - - verify(downstreamSubscriber, times(1)).onNext(item); - } - - public void testCompleteForwardsComplete() { - upstreamSubscription(); - - streamingResponseHandler.onComplete(); - - verify(downstreamSubscriber, times(1)).onSubscribe(any()); - verify(downstreamSubscriber, times(1)).onComplete(); - } - - public void testErrorForwardsError() { - var expectedError = new RetryException(false, "ah"); - upstreamSubscription(); - - streamingResponseHandler.onError(expectedError); - - verify(downstreamSubscriber, times(1)).onSubscribe(any()); - verify(downstreamSubscriber, times(1)).onError(same(expectedError)); - } - - public void testSubscriptionForwardsRequest() { - var upstreamSubscription = upstreamSubscription(); - - var downstream = ArgumentCaptor.forClass(Flow.Subscription.class); - verify(downstreamSubscriber, times(1)).onSubscribe(downstream.capture()); - var downstreamSubscription = downstream.getValue(); - - var requestCount = randomIntBetween(2, 200); - downstreamSubscription.request(requestCount); - verify(upstreamSubscription, times(1)).request(requestCount); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandlerTests.java new file mode 100644 index 0000000000000..4853aa8d2c563 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandlerTests.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.openai; + +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.elasticsearch.ExceptionsHelper.unwrapCause; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OpenAiUnifiedChatCompletionResponseHandlerTests extends ESTestCase { + private final OpenAiUnifiedChatCompletionResponseHandler responseHandler = new OpenAiUnifiedChatCompletionResponseHandler( + "chat completions", + (a, b) -> mock() + ); + + public void testFailValidationWithAllFields() throws IOException { + var responseJson = """ + { + "error": { + "type": "not_found_error", + "message": "a message", + "code": "ahh", + "param": "model" + } + } + """; + + var errorJson = invalidResponseJson(responseJson); + + assertThat(errorJson, is(""" + {"error":{"code":"ahh","message":"Received a server error status code for request from inference entity id [abc] status [500]. \ + Error message: [a message]","param":"model","type":"not_found_error"}}""")); + } + + public void testFailValidationWithoutOptionalFields() throws IOException { + var responseJson = """ + { + "error": { + "type": "not_found_error", + "message": "a message" + } + } + """; + + var errorJson = invalidResponseJson(responseJson); + + assertThat(errorJson, is(""" + {"error":{"message":"Received a server error status code for request from inference entity id [abc] status [500]. \ + Error message: [a message]","type":"not_found_error"}}""")); + } + + public void testFailValidationWithInvalidJson() throws IOException { + var responseJson = """ + what? this isn't a json + """; + + var errorJson = invalidResponseJson(responseJson); + + assertThat(errorJson, is(""" + {"error":{"code":"bad_request","message":"Received a server error status code for request from inference entity id [abc] status\ + [500]","type":"ErrorResponse"}}""")); + } + + private String invalidResponseJson(String responseJson) throws IOException { + var exception = invalidResponse(responseJson); + assertThat(exception, isA(RetryException.class)); + assertThat(unwrapCause(exception), isA(UnifiedChatCompletionException.class)); + return toJson((UnifiedChatCompletionException) unwrapCause(exception)); + } + + private Exception invalidResponse(String responseJson) { + return expectThrows( + RetryException.class, + () -> responseHandler.validateResponse( + mock(), + mock(), + mockRequest(), + new HttpResult(mock500Response(), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + } + + private static Request mockRequest() { + var request = mock(Request.class); + when(request.getInferenceEntityId()).thenReturn("abc"); + when(request.isStreaming()).thenReturn(true); + return request; + } + + private static HttpResponse mock500Response() { + int statusCode = 500; + var statusLine = mock(StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + + var response = mock(HttpResponse.class); + when(response.getStatusLine()).thenReturn(statusLine); + + return response; + } + + private String toJson(UnifiedChatCompletionException e) throws IOException { + try (var builder = XContentFactory.jsonBuilder()) { + e.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + return XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java index c0ebaf8668c5c..f81f6e58964f0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java @@ -31,7 +31,7 @@ public void testToXContent_SingleInput_UnspecifiedUsageContext() throws IOExcept assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" { "input": ["abc"], - "model_id": "my-model-id" + "model": "my-model-id" }""")); } @@ -48,7 +48,7 @@ public void testToXContent_MultipleInputs_UnspecifiedUsageContext() throws IOExc "abc", "def" ], - "model_id": "my-model-id" + "model": "my-model-id" } """)); } @@ -63,7 +63,7 @@ public void testToXContent_MultipleInputs_SearchUsageContext() throws IOExceptio assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" { "input": ["abc"], - "model_id": "my-model-id", + "model": "my-model-id", "usage_context": "search" } """)); @@ -79,7 +79,7 @@ public void testToXContent_MultipleInputs_IngestUsageContext() throws IOExceptio assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" { "input": ["abc"], - "model_id": "my-model-id", + "model": "my-model-id", "usage_context": "ingest" } """)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java index b030bab1690a7..d22a2682108ec 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java @@ -47,7 +47,7 @@ public void testCreateHttpRequest_UsageContextSearch() throws IOException { assertThat(requestMap.size(), equalTo(3)); assertThat(requestMap.get("input"), is(List.of(input))); - assertThat(requestMap.get("model_id"), is(modelId)); + assertThat(requestMap.get("model"), is(modelId)); assertThat(requestMap.get("usage_context"), equalTo("search")); } @@ -84,7 +84,7 @@ public void testTruncate_ReducesInputTextSizeByHalf() throws IOException { var requestMap = entityAsMap(httpPost.getEntity().getContent()); assertThat(requestMap, aMapWithSize(2)); assertThat(requestMap.get("input"), is(List.of("ab"))); - assertThat(requestMap.get("model_id"), is(modelId)); + assertThat(requestMap.get("model"), is(modelId)); } public void testIsTruncated_ReturnsTrue() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/rerank/IbmWatsonxRerankRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/rerank/IbmWatsonxRerankRequestEntityTests.java new file mode 100644 index 0000000000000..8278b76a1cee4 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/rerank/IbmWatsonxRerankRequestEntityTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.ibmwatsonx.rerank; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxRerankRequestEntity; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankTaskSettings; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; + +public class IbmWatsonxRerankRequestEntityTests extends ESTestCase { + public void testXContent_Request() throws IOException { + IbmWatsonxRerankTaskSettings taskSettings = new IbmWatsonxRerankTaskSettings(5, true, 100); + var entity = new IbmWatsonxRerankRequestEntity( + "database", + List.of("greenland", "google", "john", "mysql", "potter", "grammar"), + taskSettings, + "model", + "project_id" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + {"model_id":"model", + "query":"database", + "inputs":[ + {"text":"greenland"}, + {"text":"google"}, + {"text":"john"}, + {"text":"mysql"}, + {"text":"potter"}, + {"text":"grammar"} + ], + "project_id":"project_id", + "parameters":{ + "truncate_input_tokens":100, + "return_options":{ + "inputs":true, + "top_n":5 + } + } + } + """)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/rerank/IbmWatsonxRerankRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/rerank/IbmWatsonxRerankRequestTests.java new file mode 100644 index 0000000000000..8c95a01bc3230 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/rerank/IbmWatsonxRerankRequestTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.ibmwatsonx.rerank; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.core.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.request.ibmwatsonx.IbmWatsonxRerankRequest; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModelTests; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class IbmWatsonxRerankRequestTests extends ESTestCase { + private static final String AUTH_HEADER_VALUE = "foo"; + + public void testCreateRequest() throws IOException { + var model = "model"; + var projectId = "project_id"; + URI uri = null; + try { + uri = new URI("http://abc.com"); + } catch (Exception ignored) {} + var apiVersion = "2023-05-04"; + var apiKey = "api_key"; + var query = "database"; + List input = List.of("greenland", "google", "john", "mysql", "potter", "grammar"); + + var request = createRequest(model, projectId, uri, apiVersion, apiKey, query, input); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getURI().toString(), endsWith(Strings.format("%s=%s", "version", apiVersion))); + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + assertThat(requestMap, aMapWithSize(5)); + assertThat( + requestMap, + is( + + Map.of( + "project_id", + "project_id", + "model_id", + "model", + "inputs", + List.of( + Map.of("text", "greenland"), + Map.of("text", "google"), + Map.of("text", "john"), + Map.of("text", "mysql"), + Map.of("text", "potter"), + Map.of("text", "grammar") + ), + "query", + "database", + "parameters", + Map.of("return_options", Map.of("top_n", 2, "inputs", true), "truncate_input_tokens", 100) + ) + ) + ); + } + + public static IbmWatsonxRerankRequest createRequest( + String model, + String projectId, + URI uri, + String apiVersion, + String apiKey, + String query, + List input + ) { + var embeddingsModel = IbmWatsonxRerankModelTests.createModel(model, projectId, uri, apiVersion, apiKey); + + return new IbmWatsonxRerankWithoutAuthRequest(query, input, embeddingsModel); + } + + private static class IbmWatsonxRerankWithoutAuthRequest extends IbmWatsonxRerankRequest { + IbmWatsonxRerankWithoutAuthRequest(String query, List input, IbmWatsonxRerankModel model) { + super(query, input, model); + } + + @Override + public void decorateWithAuth(HttpPost httpPost) { + httpPost.setHeader(HttpHeaders.AUTHORIZATION, AUTH_HEADER_VALUE); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxRankedResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxRankedResponseEntityTests.java new file mode 100644 index 0000000000000..6b59f25896a48 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxRankedResponseEntityTests.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.inference.external.response.ibmwatsonx; + +import org.apache.http.HttpResponse; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class IbmWatsonxRankedResponseEntityTests extends ESTestCase { + + public void testResponseLiteral() throws IOException { + InferenceServiceResults parsedResults = IbmWatsonxRankedResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseLiteral.getBytes(StandardCharsets.UTF_8)) + ); + + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + List expected = responseLiteralDocs(); + for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) { + assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index()); + } + } + + public void testGeneratedResponse() throws IOException { + int numDocs = randomIntBetween(1, 10); + + List expected = new ArrayList<>(numDocs); + StringBuilder responseBuilder = new StringBuilder(); + + responseBuilder.append("{"); + responseBuilder.append("\"results\": ["); + List indices = linear(numDocs); + List scores = linearFloats(numDocs); + for (int i = 0; i < numDocs; i++) { + int index = indices.remove(randomInt(indices.size() - 1)); + + responseBuilder.append("{"); + responseBuilder.append("\"index\":").append(index).append(","); + responseBuilder.append("\"score\":").append(scores.get(i).toString()).append("}"); + expected.add(new RankedDocsResults.RankedDoc(index, scores.get(i), null)); + if (i < numDocs - 1) { + responseBuilder.append(","); + } + } + responseBuilder.append("]"); + + responseBuilder.append(randomIntBetween(1, 10)).append("}"); + + InferenceServiceResults parsedResults = IbmWatsonxRankedResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseBuilder.toString().getBytes(StandardCharsets.UTF_8)) + ); + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) { + assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index()); + } + } + + private ArrayList responseLiteralDocs() { + var list = new ArrayList(); + + list.add(new RankedDocsResults.RankedDoc(2, 0.98005307F, null)); + list.add(new RankedDocsResults.RankedDoc(3, 0.27904198F, null)); + list.add(new RankedDocsResults.RankedDoc(0, 0.10194652F, null)); + return list; + } + + private final String responseLiteral = """ + { + "results": [ + { + "index": 2, + "score": 0.98005307 + }, + { + "index": 3, + "score": 0.27904198 + }, + { + "index": 0, + "score": 0.10194652 + } + ] + } + """; + + public void testResponseLiteralWithDocuments() throws IOException { + InferenceServiceResults parsedResults = IbmWatsonxRankedResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseLiteralWithDocuments.getBytes(StandardCharsets.UTF_8)) + ); + + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + MatcherAssert.assertThat(((RankedDocsResults) parsedResults).getRankedDocs(), is(responseLiteralDocsWithText)); + } + + private final String responseLiteralWithDocuments = """ + { + "results": [ + { + "input": { + "text": "Washington, D.C.." + }, + "index": 2, + "score": 0.98005307 + }, + { + "input": { + "text": "Capital punishment has existed in the United States since before the United States was a country. " + }, + "index": 3, + "score": 0.27904198 + }, + { + "input": { + "text": "Carson City is the capital city of the American state of Nevada." + }, + "index": 0, + "score": 0.10194652 + } + ] + } + """; + + private final List responseLiteralDocsWithText = List.of( + new RankedDocsResults.RankedDoc(2, 0.98005307F, "Washington, D.C.."), + new RankedDocsResults.RankedDoc( + 3, + 0.27904198F, + "Capital punishment has existed in the United States since before the United States was a country. " + ), + new RankedDocsResults.RankedDoc(0, 0.10194652F, "Carson City is the capital city of the American state of Nevada.") + ); + + private ArrayList linear(int n) { + ArrayList list = new ArrayList<>(); + for (int i = 0; i <= n; i++) { + list.add(i); + } + return list; + } + + // creates a list of doubles of monotonically decreasing magnitude + private ArrayList linearFloats(int n) { + ArrayList list = new ArrayList<>(); + float startValue = 1.0f; + float decrement = startValue / n + 1; + for (int i = 0; i <= n; i++) { + list.add(startValue - (i * decrement)); + } + return list; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java index 2fa6d520fcc6f..d1fbf110dad8c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java @@ -73,9 +73,7 @@ protected List extraMappers() { @Override protected Settings indexSettings() { - var builder = Settings.builder() - .put(super.indexSettings()) - .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false); + var builder = Settings.builder().put(super.indexSettings()); if (useSynthetic) { builder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); builder.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true); @@ -222,7 +220,9 @@ private Translog.Snapshot newRandomSnapshot( private static Model randomModel(TaskType taskType) { var dimensions = taskType == TaskType.TEXT_EMBEDDING ? randomIntBetween(2, 64) : null; - var similarity = taskType == TaskType.TEXT_EMBEDDING ? randomFrom(SimilarityMeasure.values()) : null; + var similarity = taskType == TaskType.TEXT_EMBEDDING + ? randomValueOtherThan(SimilarityMeasure.DOT_PRODUCT, () -> randomFrom(SimilarityMeasure.values())) + : null; var elementType = taskType == TaskType.TEXT_EMBEDDING ? DenseVectorFieldMapper.ElementType.BYTE : null; return new TestModel( randomAlphaOfLength(4), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java index 24183b21f73e7..d196efa0d152b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java @@ -7,7 +7,9 @@ package org.elasticsearch.xpack.inference.mapper; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.NonDynamicFieldMapperTests; +import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; import org.elasticsearch.xpack.inference.Utils; @@ -25,6 +27,11 @@ public void setup() throws Exception { Utils.storeSparseModel(client()); } + @Override + protected Settings nodeSettings() { + return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build(); + } + @Override protected Collection> getPlugins() { return List.of(LocalStateInferencePlugin.class); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java index 162bcc8f09713..65e4d049ef58b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchResponseUtils; @@ -41,6 +42,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.elasticsearch.core.Strings.format; @@ -52,6 +54,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ModelRegistryTests extends ESTestCase { @@ -295,6 +299,37 @@ public void testStoreModel_ThrowsException_WhenFailureIsNotAVersionConflict() { ); } + public void testRemoveDefaultConfigs_DoesNotCallClient_WhenPassedAnEmptySet() { + var client = mock(Client.class); + + var registry = new ModelRegistry(client); + var listener = new PlainActionFuture(); + + registry.removeDefaultConfigs(Set.of(), listener); + + assertTrue(listener.actionGet(TIMEOUT)); + verify(client, times(0)).execute(any(), any(), any()); + } + + public void testDeleteModels_Returns_ConflictException_WhenModelIsBeingAdded() { + var client = mockClient(); + + var registry = new ModelRegistry(client); + var model = TestModel.createRandomInstance(); + var newModel = TestModel.createRandomInstance(); + registry.updateModelTransaction(newModel, model, new PlainActionFuture<>()); + + var listener = new PlainActionFuture(); + + registry.deleteModels(Set.of(newModel.getInferenceEntityId()), listener); + var exception = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + exception.getMessage(), + containsString("are currently being updated, please wait until after they are finished updating to delete.") + ); + assertThat(exception.status(), is(RestStatus.CONFLICT)); + } + public void testIdMatchedDefault() { var defaultConfigIds = new ArrayList(); defaultConfigIds.add( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java index 5528c80066b0a..4961778a03726 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.test.rest.RestActionTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.junit.Before; @@ -42,6 +43,11 @@ public class BaseInferenceActionTests extends RestActionTestCase { @Before public void setUpAction() { controller().registerHandler(new BaseInferenceAction() { + @Override + protected boolean shouldStream() { + return false; + } + @Override protected ActionListener listener(RestChannel channel) { return new RestChunkedToXContentListener<>(channel); @@ -102,10 +108,10 @@ public void testParseTimeout_ReturnsDefaultTimeout() { public void testUsesDefaultTimeout() { SetOnce executeCalled = new SetOnce<>(); verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { - assertThat(actionRequest, instanceOf(InferenceAction.Request.class)); + assertThat(actionRequest, instanceOf(InferenceActionProxy.Request.class)); - var request = (InferenceAction.Request) actionRequest; - assertThat(request.getInferenceTimeout(), is(InferenceAction.Request.DEFAULT_TIMEOUT)); + var request = (InferenceActionProxy.Request) actionRequest; + assertThat(request.getTimeout(), is(InferenceAction.Request.DEFAULT_TIMEOUT)); executeCalled.set(true); return createResponse(); @@ -122,10 +128,10 @@ public void testUsesDefaultTimeout() { public void testUses3SecondTimeoutFromParams() { SetOnce executeCalled = new SetOnce<>(); verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { - assertThat(actionRequest, instanceOf(InferenceAction.Request.class)); + assertThat(actionRequest, instanceOf(InferenceActionProxy.Request.class)); - var request = (InferenceAction.Request) actionRequest; - assertThat(request.getInferenceTimeout(), is(TimeValue.timeValueSeconds(3))); + var request = (InferenceActionProxy.Request) actionRequest; + assertThat(request.getTimeout(), is(TimeValue.timeValueSeconds(3))); executeCalled.set(true); return createResponse(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionProxyTests.java similarity index 90% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionProxyTests.java index 1b0df1b4a20da..433e33fe15210 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionProxyTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.RestActionTestCase; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; import org.junit.Before; import static org.elasticsearch.xpack.inference.rest.BaseInferenceActionTests.createResponse; @@ -21,7 +21,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class RestInferenceActionTests extends RestActionTestCase { +public class RestInferenceActionProxyTests extends RestActionTestCase { @Before public void setUpAction() { @@ -31,9 +31,9 @@ public void setUpAction() { public void testStreamIsFalse() { SetOnce executeCalled = new SetOnce<>(); verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { - assertThat(actionRequest, instanceOf(InferenceAction.Request.class)); + assertThat(actionRequest, instanceOf(InferenceActionProxy.Request.class)); - var request = (InferenceAction.Request) actionRequest; + var request = (InferenceActionProxy.Request) actionRequest; assertThat(request.isStreaming(), is(false)); executeCalled.set(true); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java index f67680ef6b625..e69dd3fda6240 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestStreamInferenceActionTests.java @@ -9,13 +9,18 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.rest.AbstractRestChannel; +import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.RestActionTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; import org.junit.After; import org.junit.Before; @@ -42,9 +47,9 @@ public void tearDownAction() { public void testStreamIsTrue() { SetOnce executeCalled = new SetOnce<>(); verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { - assertThat(actionRequest, instanceOf(InferenceAction.Request.class)); + assertThat(actionRequest, instanceOf(InferenceActionProxy.Request.class)); - var request = (InferenceAction.Request) actionRequest; + var request = (InferenceActionProxy.Request) actionRequest; assertThat(request.isStreaming(), is(true)); executeCalled.set(true); @@ -58,4 +63,50 @@ public void testStreamIsTrue() { dispatchRequest(inferenceRequest); assertThat(executeCalled.get(), equalTo(true)); } + + public void testStreamIsTrue_ChatCompletion() { + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { + assertThat(actionRequest, instanceOf(InferenceActionProxy.Request.class)); + + var request = (InferenceActionProxy.Request) actionRequest; + assertThat(request.isStreaming(), is(true)); + + executeCalled.set(true); + return createResponse(); + })); + + var requestBody = """ + { + "messages": [ + { + "content": "abc", + "role": "user" + } + ] + } + """; + + RestRequest inferenceRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) + .withPath("_inference/chat_completion/test/_stream") + .withContent(new BytesArray(requestBody), XContentType.JSON) + .build(); + + final SetOnce responseSetOnce = new SetOnce<>(); + dispatchRequest(inferenceRequest, new AbstractRestChannel(inferenceRequest, true) { + @Override + public void sendResponse(RestResponse response) { + responseSetOnce.set(response); + } + }); + + // the response content will be null when there is no error + assertNull(responseSetOnce.get().content()); + assertThat(executeCalled.get(), equalTo(true)); + } + + private void dispatchRequest(final RestRequest request, final RestChannel channel) { + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + controller().dispatchRequest(request, channel, threadContext); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java deleted file mode 100644 index 9dc23c890c14d..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.rest; - -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.rest.AbstractRestChannel; -import org.elasticsearch.rest.RestChannel; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; -import org.junit.After; -import org.junit.Before; - -import static org.elasticsearch.xpack.inference.rest.BaseInferenceActionTests.createResponse; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; - -public class RestUnifiedCompletionInferenceActionTests extends RestActionTestCase { - private final SetOnce threadPool = new SetOnce<>(); - - @Before - public void setUpAction() { - threadPool.set(new TestThreadPool(getTestName())); - controller().registerHandler(new RestUnifiedCompletionInferenceAction(threadPool)); - } - - @After - public void tearDownAction() { - terminate(threadPool.get()); - } - - public void testStreamIsTrue() { - SetOnce executeCalled = new SetOnce<>(); - verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { - assertThat(actionRequest, instanceOf(UnifiedCompletionAction.Request.class)); - - var request = (UnifiedCompletionAction.Request) actionRequest; - assertThat(request.isStreaming(), is(true)); - - executeCalled.set(true); - return createResponse(); - })); - - var requestBody = """ - { - "messages": [ - { - "content": "abc", - "role": "user" - } - ] - } - """; - - RestRequest inferenceRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) - .withPath("_inference/completion/test/_unified") - .withContent(new BytesArray(requestBody), XContentType.JSON) - .build(); - - final SetOnce responseSetOnce = new SetOnce<>(); - dispatchRequest(inferenceRequest, new AbstractRestChannel(inferenceRequest, true) { - @Override - public void sendResponse(RestResponse response) { - responseSetOnce.set(response); - } - }); - - // the response content will be null when there is no error - assertNull(responseSetOnce.get().content()); - assertThat(executeCalled.get(), equalTo(true)); - } - - private void dispatchRequest(final RestRequest request, final RestChannel channel) { - ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - controller().dispatchRequest(request, channel, threadContext); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java index 7cfd231be39f3..637ae726572a4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xcontent.XContentFactory; import org.hamcrest.MatcherAssert; @@ -85,6 +86,16 @@ public InferenceEventsAssertion hasErrorContaining(String message) { return this; } + public InferenceEventsAssertion hasErrorMatching(CheckedConsumer matcher) { + hasError(); + try { + matcher.accept(error); + } catch (Exception e) { + fail(e); + } + return this; + } + public InferenceEventsAssertion hasEvents(String... events) { Arrays.stream(events).forEach(this::hasEvent); return this; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java index 92544d5535acb..1ca50d1887ee1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java @@ -490,7 +490,7 @@ public void testGetConfiguration() throws Exception { "http_schema": { "description": "", "label": "HTTP Schema", - "required": true, + "required": false, "sensitive": false, "updatable": false, "type": "str", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index e65b4f1279445..83ef46e08afbf 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -154,69 +154,80 @@ public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOExcepti @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createAmazonBedrockService()) { - String content = XContentHelper.stripWhitespace(""" - { - "service": "amazonbedrock", - "name": "Amazon Bedrock", - "task_types": ["text_embedding", "completion"], - "configurations": { - "secret_key": { - "description": "A valid AWS secret key that is paired with the access_key.", - "label": "Secret Key", - "required": true, - "sensitive": true, - "updatable": true, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "provider": { - "description": "The model provider for your deployment.", - "label": "Provider", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "access_key": { - "description": "A valid AWS access key that has permissions to use Amazon Bedrock.", - "label": "Access Key", - "required": true, - "sensitive": true, - "updatable": true, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "model": { - "description": "The base model ID or an ARN to a custom model based on a foundational model.", - "label": "Model", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "rate_limit.requests_per_minute": { - "description": "By default, the amazonbedrock service sets the number of requests allowed per minute to 240.", - "label": "Rate Limit", - "required": false, - "sensitive": false, - "updatable": false, - "type": "int", - "supported_task_types": ["text_embedding", "completion"] - }, - "region": { - "description": "The region that your model or ARN is deployed in.", - "label": "Region", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] + String content = XContentHelper.stripWhitespace( + """ + { + "service": "amazonbedrock", + "name": "Amazon Bedrock", + "task_types": ["text_embedding", "completion"], + "configurations": { + "dimensions": { + "description": "The number of dimensions the resulting embeddings should have. For more information refer to https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html.", + "label": "Dimensions", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding"] + }, + "secret_key": { + "description": "A valid AWS secret key that is paired with the access_key.", + "label": "Secret Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "provider": { + "description": "The model provider for your deployment.", + "label": "Provider", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "access_key": { + "description": "A valid AWS access key that has permissions to use Amazon Bedrock.", + "label": "Access Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "model": { + "description": "The base model ID or an ARN to a custom model based on a foundational model.", + "label": "Model", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "rate_limit.requests_per_minute": { + "description": "By default, the amazonbedrock service sets the number of requests allowed per minute to 240.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding", "completion"] + }, + "region": { + "description": "The region that your model or ARN is deployed in.", + "label": "Region", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + } } } - } - """); + """ + ); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java index 33101a3e02661..45a5c59302cf6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java @@ -553,13 +553,11 @@ public void testInfer_StreamRequest() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + streamChatCompletion().hasNoErrors().hasEvent(""" {"completion":[{"delta":"Hello"},{"delta":", World"}]}"""); } - private InferenceServiceResults streamChatCompletion() throws IOException { + private InferenceEventsAssertion streamChatCompletion() throws Exception { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new AnthropicService(senderFactory, createWithEmptySettings(threadPool))) { var model = AnthropicChatCompletionModelTests.createChatCompletionModel( @@ -580,7 +578,7 @@ private InferenceServiceResults streamChatCompletion() throws IOException { listener ); - return listener.actionGet(TIMEOUT); + return InferenceEventsAssertion.assertThat(listener.actionGet(TIMEOUT)).hasFinishedStream(); } } @@ -591,11 +589,7 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result) - .hasFinishedStream() - .hasNoEvents() + streamChatCompletion().hasNoEvents() .hasErrorWithStatusCode(RestStatus.REQUEST_ENTITY_TOO_LARGE.getStatus()) .hasErrorContaining("blah"); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java index c96b71e66f37d..ef700bcc84eed 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java @@ -60,7 +60,6 @@ import org.junit.Before; import java.io.IOException; -import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -1332,13 +1331,11 @@ public void testInfer_StreamRequest() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + streamChatCompletion().hasNoErrors().hasEvent(""" {"completion":[{"delta":"hello, world"}]}"""); } - private InferenceServiceResults streamChatCompletion() throws IOException, URISyntaxException { + private InferenceEventsAssertion streamChatCompletion() throws Exception { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new AzureAiStudioService(senderFactory, createWithEmptySettings(threadPool))) { var model = AzureAiStudioChatCompletionModelTests.createModel( @@ -1360,7 +1357,7 @@ private InferenceServiceResults streamChatCompletion() throws IOException, URISy listener ); - return listener.actionGet(TIMEOUT); + return InferenceEventsAssertion.assertThat(listener.actionGet(TIMEOUT)).hasFinishedStream(); } } @@ -1376,72 +1373,84 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { }"""; webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result) - .hasFinishedStream() - .hasNoEvents() - .hasErrorWithStatusCode(401) - .hasErrorContaining("You didn't provide an API key..."); + var e = assertThrows(ElasticsearchStatusException.class, this::streamChatCompletion); + assertThat( + e.getMessage(), + equalTo( + "Received an authentication error status code for request from inference entity id [id] status [401]. " + + "Error message: [You didn't provide an API key...]" + ) + ); } @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createService()) { - String content = XContentHelper.stripWhitespace(""" - { - "service": "azureaistudio", - "name": "Azure AI Studio", - "task_types": ["text_embedding", "completion"], - "configurations": { - "endpoint_type": { - "description": "Specifies the type of endpoint that is used in your model deployment.", - "label": "Endpoint Type", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "provider": { - "description": "The model provider for your deployment.", - "label": "Provider", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "api_key": { - "description": "API Key for the provider you're connecting to.", - "label": "API Key", - "required": true, - "sensitive": true, - "updatable": true, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] - }, - "rate_limit.requests_per_minute": { - "description": "Minimize the number of rate limit errors.", - "label": "Rate Limit", - "required": false, - "sensitive": false, - "updatable": false, - "type": "int", - "supported_task_types": ["text_embedding", "completion"] - }, - "target": { - "description": "The target URL of your Azure AI Studio model deployment.", - "label": "Target", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion"] + String content = XContentHelper.stripWhitespace( + """ + { + "service": "azureaistudio", + "name": "Azure AI Studio", + "task_types": ["text_embedding", "completion"], + "configurations": { + "dimensions": { + "description": "The number of dimensions the resulting embeddings should have. For more information refer to https://learn.microsoft.com/en-us/azure/ai-studio/reference/reference-model-inference-embeddings.", + "label": "Dimensions", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding"] + }, + "endpoint_type": { + "description": "Specifies the type of endpoint that is used in your model deployment.", + "label": "Endpoint Type", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "provider": { + "description": "The model provider for your deployment.", + "label": "Provider", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "api_key": { + "description": "API Key for the provider you're connecting to.", + "label": "API Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + }, + "rate_limit.requests_per_minute": { + "description": "Minimize the number of rate limit errors.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding", "completion"] + }, + "target": { + "description": "The target URL of your Azure AI Studio model deployment.", + "label": "Target", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "completion"] + } } } - } - """); + """ + ); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index 84e04fc315cd0..85caff9048434 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -1397,13 +1397,11 @@ public void testInfer_StreamRequest() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + streamChatCompletion().hasNoErrors().hasEvent(""" {"completion":[{"delta":"hello, world"}]}"""); } - private InferenceServiceResults streamChatCompletion() throws IOException, URISyntaxException { + private InferenceEventsAssertion streamChatCompletion() throws Exception { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new AzureOpenAiService(senderFactory, createWithEmptySettings(threadPool))) { var model = AzureOpenAiCompletionModelTests.createCompletionModel( @@ -1428,7 +1426,7 @@ private InferenceServiceResults streamChatCompletion() throws IOException, URISy listener ); - return listener.actionGet(TIMEOUT); + return InferenceEventsAssertion.assertThat(listener.actionGet(TIMEOUT)).hasFinishedStream(); } } @@ -1444,13 +1442,14 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { }"""; webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result) - .hasFinishedStream() - .hasNoEvents() - .hasErrorWithStatusCode(401) - .hasErrorContaining("You didn't provide an API key..."); + var e = assertThrows(ElasticsearchStatusException.class, this::streamChatCompletion); + assertThat( + e.getMessage(), + equalTo( + "Received an authentication error status code for request from inference entity id [id] status [401]." + + " Error message: [You didn't provide an API key...]" + ) + ); } @SuppressWarnings("checkstyle:LineLength") @@ -1472,6 +1471,15 @@ public void testGetConfiguration() throws Exception { "type": "str", "supported_task_types": ["text_embedding", "completion"] }, + "dimensions": { + "description": "The number of dimensions the resulting embeddings should have. For more information refer to https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-body-1.", + "label": "Dimensions", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding"] + }, "entra_id": { "description": "You must provide either an API key or an Entra ID.", "label": "Entra ID", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index be581badca057..7916df2536dc7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -1577,7 +1577,7 @@ public void testChunkedInfer_BatchesCalls_Bytes() throws IOException { } public void testDefaultSimilarity() { - assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity()); + assertEquals(SimilarityMeasure.COSINE, CohereService.defaultSimilarity()); } public void testInfer_StreamRequest() throws Exception { @@ -1587,13 +1587,11 @@ public void testInfer_StreamRequest() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + streamChatCompletion().hasNoErrors().hasEvent(""" {"completion":[{"delta":"hello"},{"delta":"there"}]}"""); } - private InferenceServiceResults streamChatCompletion() throws IOException { + private InferenceEventsAssertion streamChatCompletion() throws Exception { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { var model = CohereCompletionModelTests.createModel(getUrl(webServer), "secret", "model"); @@ -1609,7 +1607,7 @@ private InferenceServiceResults streamChatCompletion() throws IOException { listener ); - return listener.actionGet(TIMEOUT); + return InferenceEventsAssertion.assertThat(listener.actionGet(TIMEOUT)).hasFinishedStream(); } } @@ -1619,13 +1617,7 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamChatCompletion(); - - InferenceEventsAssertion.assertThat(result) - .hasFinishedStream() - .hasNoEvents() - .hasErrorWithStatusCode(500) - .hasErrorContaining("how dare you"); + streamChatCompletion().hasNoEvents().hasErrorWithStatusCode(500).hasErrorContaining("how dare you"); } @SuppressWarnings("checkstyle:LineLength") @@ -1646,6 +1638,15 @@ public void testGetConfiguration() throws Exception { "type": "str", "supported_task_types": ["text_embedding", "rerank", "completion"] }, + "model_id": { + "description": "The name of the model to use for the inference task.", + "label": "Model ID", + "required": false, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "rerank", "completion"] + }, "rate_limit.requests_per_minute": { "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java index e477ffb10def0..2616393ac8442 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.services.elastic; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; @@ -17,6 +18,30 @@ public class ElasticInferenceServiceSettingsTests extends ESTestCase { private static final String ELASTIC_INFERENCE_SERVICE_URL = "http://elastic-inference-service"; private static final String ELASTIC_INFERENCE_SERVICE_LEGACY_URL = "http://elastic-inference-service-legacy"; + public static ElasticInferenceServiceSettings create(String elasticInferenceServiceUrl) { + var settings = Settings.builder() + .put(ElasticInferenceServiceSettings.ELASTIC_INFERENCE_SERVICE_URL.getKey(), elasticInferenceServiceUrl) + .build(); + + return new ElasticInferenceServiceSettings(settings); + } + + public static ElasticInferenceServiceSettings create( + String elasticInferenceServiceUrl, + TimeValue authorizationRequestInterval, + TimeValue maxJitter, + boolean periodicAuthorizationEnabled + ) { + var settings = Settings.builder() + .put(ElasticInferenceServiceSettings.ELASTIC_INFERENCE_SERVICE_URL.getKey(), elasticInferenceServiceUrl) + .put(ElasticInferenceServiceSettings.AUTHORIZATION_REQUEST_INTERVAL.getKey(), authorizationRequestInterval) + .put(ElasticInferenceServiceSettings.MAX_AUTHORIZATION_REQUEST_JITTER.getKey(), maxJitter) + .put(ElasticInferenceServiceSettings.PERIODIC_AUTHORIZATION_ENABLED.getKey(), periodicAuthorizationEnabled) + .build(); + + return new ElasticInferenceServiceSettings(settings); + } + public void testGetElasticInferenceServiceUrl_WithUrlSetting() { var settings = Settings.builder() .put(ElasticInferenceServiceSettings.ELASTIC_INFERENCE_SERVICE_URL.getKey(), ELASTIC_INFERENCE_SERVICE_URL) @@ -53,5 +78,4 @@ public void testGetElasticInferenceServiceUrl_WithoutUrlSetting() { assertThat(eisSettings.getElasticInferenceServiceUrl(), equalTo("")); } - } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java index 02bbbb844c04f..4bd673e856123 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java @@ -26,7 +26,7 @@ public static ElasticInferenceServiceSparseEmbeddingsModel createModel(String ur new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, maxInputTokens, null), EmptyTaskSettings.INSTANCE, EmptySecretSettings.INSTANCE, - new ElasticInferenceServiceComponents(url) + ElasticInferenceServiceComponents.of(url) ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index 15e40d40eb59b..cf90015aa81c3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -27,14 +27,17 @@ import org.elasticsearch.inference.MinimalServiceSettings; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.http.MockResponse; import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -44,11 +47,15 @@ import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests; +import org.elasticsearch.xpack.inference.services.InferenceEventsAssertion; import org.elasticsearch.xpack.inference.services.ServiceFields; -import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorization; -import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationHandler; -import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationTests; +import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationModel; +import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationModelTests; +import org.elasticsearch.xpack.inference.services.elastic.authorization.ElasticInferenceServiceAuthorizationRequestHandler; +import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionModel; +import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.After; @@ -61,8 +68,10 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.ExceptionsHelper.unwrapCause; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; import static org.elasticsearch.xpack.inference.Utils.getModelListenerForException; import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; @@ -76,6 +85,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isA; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -354,8 +364,20 @@ public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException } private ModelRegistry mockModelRegistry() { + return mockModelRegistry(threadPool); + } + + public static ModelRegistry mockModelRegistry(ThreadPool threadPool) { var client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); + + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArgument(2); + listener.onResponse(true); + + return Void.TYPE; + }).when(client).execute(any(), any(), any()); return new ModelRegistry(client); } @@ -426,7 +448,7 @@ public void testInfer_ThrowsErrorWhenTaskTypeIsNotValid_ChatCompletion() throws "Inference entity [model_id] does not support task type [chat_completion] " + "for inference, the task type must be one of [sparse_embedding]. " + "The task type for the inference entity is chat_completion, " - + "please use the _inference/chat_completion/model_id/_unified URL." + + "please use the _inference/chat_completion/model_id/_stream URL." ) ); @@ -486,8 +508,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { assertThat(request.getHeader(HttpHeaders.CONTENT_TYPE), Matchers.equalTo(XContentType.JSON.mediaType())); var requestMap = entityAsMap(request.getBody()); - - assertThat(requestMap, is(Map.of("input", List.of("input text"), "model_id", "my-model-id", "usage_context", "search"))); + assertThat(requestMap, is(Map.of("input", List.of("input text"), "model", "my-model-id", "usage_context", "search"))); } } @@ -544,13 +565,14 @@ public void testChunkedInfer_PassesThrough() throws IOException { ); var requestMap = entityAsMap(webServer.requests().get(0).getBody()); - - assertThat(requestMap, is(Map.of("input", List.of("input text"), "model_id", "my-model-id", "usage_context", "ingest"))); + assertThat(requestMap, is(Map.of("input", List.of("input text"), "model", "my-model-id", "usage_context", "ingest"))); } } public void testHideFromConfigurationApi_ReturnsTrue_WithNoAvailableModels() throws Exception { - try (var service = createServiceWithMockSender(ElasticInferenceServiceAuthorization.newDisabledService())) { + try (var service = createServiceWithMockSender(ElasticInferenceServiceAuthorizationModel.newDisabledService())) { + ensureAuthorizationCallFinished(service); + assertTrue(service.hideFromConfigurationApi()); } } @@ -558,7 +580,7 @@ public void testHideFromConfigurationApi_ReturnsTrue_WithNoAvailableModels() thr public void testHideFromConfigurationApi_ReturnsTrue_WithModelTaskTypesThatAreNotImplemented() throws Exception { try ( var service = createServiceWithMockSender( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -570,6 +592,8 @@ public void testHideFromConfigurationApi_ReturnsTrue_WithModelTaskTypesThatAreNo ) ) ) { + ensureAuthorizationCallFinished(service); + assertTrue(service.hideFromConfigurationApi()); } } @@ -577,7 +601,7 @@ public void testHideFromConfigurationApi_ReturnsTrue_WithModelTaskTypesThatAreNo public void testHideFromConfigurationApi_ReturnsFalse_WithAvailableModels() throws Exception { try ( var service = createServiceWithMockSender( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -589,6 +613,8 @@ public void testHideFromConfigurationApi_ReturnsFalse_WithAvailableModels() thro ) ) ) { + ensureAuthorizationCallFinished(service); + assertFalse(service.hideFromConfigurationApi()); } } @@ -596,7 +622,7 @@ public void testHideFromConfigurationApi_ReturnsFalse_WithAvailableModels() thro public void testGetConfiguration() throws Exception { try ( var service = createServiceWithMockSender( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -608,6 +634,8 @@ public void testGetConfiguration() throws Exception { ) ) ) { + ensureAuthorizationCallFinished(service); + String content = XContentHelper.stripWhitespace(""" { "service": "elastic", @@ -660,7 +688,9 @@ public void testGetConfiguration() throws Exception { } public void testGetConfiguration_WithoutSupportedTaskTypes() throws Exception { - try (var service = createServiceWithMockSender(ElasticInferenceServiceAuthorization.newDisabledService())) { + try (var service = createServiceWithMockSender(ElasticInferenceServiceAuthorizationModel.newDisabledService())) { + ensureAuthorizationCallFinished(service); + String content = XContentHelper.stripWhitespace(""" { "service": "elastic", @@ -716,7 +746,7 @@ public void testGetConfiguration_WithoutSupportedTaskTypes_WhenModelsReturnTaskO try ( var service = createServiceWithMockSender( // this service doesn't yet support text embedding so we should still have no task types - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -728,6 +758,8 @@ public void testGetConfiguration_WithoutSupportedTaskTypes_WhenModelsReturnTaskO ) ) ) { + ensureAuthorizationCallFinished(service); + String content = XContentHelper.stripWhitespace(""" { "service": "elastic", @@ -795,7 +827,8 @@ public void testSupportedStreamingTasks_ReturnsChatCompletion_WhenAuthRespondsWi var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createServiceWithAuthHandler(senderFactory, getUrl(webServer))) { - service.waitForAuthorizationToComplete(TIMEOUT); + ensureAuthorizationCallFinished(service); + assertThat(service.supportedStreamingTasks(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.ANY))); assertTrue(service.defaultConfigIds().isEmpty()); @@ -825,7 +858,8 @@ public void testSupportedTaskTypes_Returns_TheAuthorizedTaskTypes_IgnoresUnimple var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createServiceWithAuthHandler(senderFactory, getUrl(webServer))) { - service.waitForAuthorizationToComplete(TIMEOUT); + ensureAuthorizationCallFinished(service); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING))); } } @@ -850,7 +884,8 @@ public void testSupportedTaskTypes_Returns_TheAuthorizedTaskTypes() throws Excep var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createServiceWithAuthHandler(senderFactory, getUrl(webServer))) { - service.waitForAuthorizationToComplete(TIMEOUT); + ensureAuthorizationCallFinished(service); + assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION))); } } @@ -871,7 +906,8 @@ public void testSupportedStreamingTasks_ReturnsEmpty_WhenAuthRespondsWithoutChat var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createServiceWithAuthHandler(senderFactory, getUrl(webServer))) { - service.waitForAuthorizationToComplete(TIMEOUT); + ensureAuthorizationCallFinished(service); + assertThat(service.supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class))); assertTrue(service.defaultConfigIds().isEmpty()); assertThat(service.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING))); @@ -898,7 +934,7 @@ public void testDefaultConfigs_Returns_DefaultChatCompletion_V1_WhenTaskTypeIsIn var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createServiceWithAuthHandler(senderFactory, getUrl(webServer))) { - service.waitForAuthorizationToComplete(TIMEOUT); + ensureAuthorizationCallFinished(service); assertThat(service.supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class))); assertThat( service.defaultConfigIds(), @@ -932,7 +968,7 @@ public void testDefaultConfigs_Returns_DefaultChatCompletion_V1_WhenTaskTypeIsCo var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createServiceWithAuthHandler(senderFactory, getUrl(webServer))) { - service.waitForAuthorizationToComplete(TIMEOUT); + ensureAuthorizationCallFinished(service); assertThat(service.supportedStreamingTasks(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.ANY))); assertThat( service.defaultConfigIds(), @@ -950,43 +986,140 @@ public void testDefaultConfigs_Returns_DefaultChatCompletion_V1_WhenTaskTypeIsCo } } + public void testUnifiedCompletionError() { + var e = assertThrows(UnifiedChatCompletionException.class, () -> testUnifiedStream(404, """ + { + "error": "The model `rainbow-sprinkles` does not exist or you do not have access to it." + }""")); + assertThat( + e.getMessage(), + equalTo( + "Received an unsuccessful status code for request from inference entity id [id] status " + + "[404]. Error message: [The model `rainbow-sprinkles` does not exist or you do not have access to it.]" + ) + ); + } + + public void testUnifiedCompletionErrorMidStream() throws Exception { + testUnifiedStreamError(200, """ + data: { "error": "some error" } + + """, """ + {\ + "error":{\ + "code":"stream_error",\ + "message":"Received an error response for request from inference entity id [id]. Error message: [some error]",\ + "type":"error"\ + }}"""); + } + + public void testUnifiedCompletionMalformedError() throws Exception { + testUnifiedStreamError(200, """ + data: { i am not json } + + """, """ + {\ + "error":{\ + "code":"bad_request",\ + "message":"[1:3] Unexpected character ('i' (code 105)): was expecting double-quote to start field name\\n\ + at [Source: (String)\\"{ i am not json }\\"; line: 1, column: 3]",\ + "type":"x_content_parse_exception"\ + }}"""); + } + + private void testUnifiedStreamError(int responseCode, String responseJson, String expectedJson) throws Exception { + testUnifiedStream(responseCode, responseJson).hasNoEvents().hasErrorMatching(e -> { + e = unwrapCause(e); + assertThat(e, isA(UnifiedChatCompletionException.class)); + try (var builder = XContentFactory.jsonBuilder()) { + ((UnifiedChatCompletionException) e).toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + var json = XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); + + assertThat(json, is(expectedJson)); + } + }); + } + + private InferenceEventsAssertion testUnifiedStream(int responseCode, String responseJson) throws Exception { + var eisGatewayUrl = getUrl(webServer); + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = createService(senderFactory, eisGatewayUrl)) { + webServer.enqueue(new MockResponse().setResponseCode(responseCode).setBody(responseJson)); + var model = new ElasticInferenceServiceCompletionModel( + "id", + TaskType.COMPLETION, + "elastic", + new ElasticInferenceServiceCompletionServiceSettings("model_id", new RateLimitSettings(100)), + EmptyTaskSettings.INSTANCE, + EmptySecretSettings.INSTANCE, + ElasticInferenceServiceComponents.of(eisGatewayUrl) + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.unifiedCompletionInfer( + model, + UnifiedCompletionRequest.of( + List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "user", null, null)) + ), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + return InferenceEventsAssertion.assertThat(listener.actionGet(TIMEOUT)).hasFinishedStream(); + } + } + + private void ensureAuthorizationCallFinished(ElasticInferenceService service) { + service.onNodeStarted(); + service.waitForFirstAuthorizationToComplete(TIMEOUT); + } + private ElasticInferenceService createServiceWithMockSender() { - return createServiceWithMockSender(ElasticInferenceServiceAuthorizationTests.createEnabledAuth()); + return createServiceWithMockSender(ElasticInferenceServiceAuthorizationModelTests.createEnabledAuth()); } - private ElasticInferenceService createServiceWithMockSender(ElasticInferenceServiceAuthorization auth) { - var mockAuthHandler = mock(ElasticInferenceServiceAuthorizationHandler.class); + private ElasticInferenceService createServiceWithMockSender(ElasticInferenceServiceAuthorizationModel auth) { + var mockAuthHandler = mock(ElasticInferenceServiceAuthorizationRequestHandler.class); doAnswer(invocation -> { - ActionListener listener = invocation.getArgument(0); + ActionListener listener = invocation.getArgument(0); listener.onResponse(auth); return Void.TYPE; }).when(mockAuthHandler).getAuthorization(any(), any()); + var sender = mock(Sender.class); + + var factory = mock(HttpRequestSender.Factory.class); + when(factory.createSender()).thenReturn(sender); return new ElasticInferenceService( - mock(HttpRequestSender.Factory.class), + factory, createWithEmptySettings(threadPool), - new ElasticInferenceServiceComponents(null), + new ElasticInferenceServiceSettings(Settings.EMPTY), mockModelRegistry(), mockAuthHandler ); } private ElasticInferenceService createService(HttpRequestSender.Factory senderFactory) { - return createService(senderFactory, ElasticInferenceServiceAuthorizationTests.createEnabledAuth(), null); + return createService(senderFactory, ElasticInferenceServiceAuthorizationModelTests.createEnabledAuth(), null); } private ElasticInferenceService createService(HttpRequestSender.Factory senderFactory, String gatewayUrl) { - return createService(senderFactory, ElasticInferenceServiceAuthorizationTests.createEnabledAuth(), gatewayUrl); + return createService(senderFactory, ElasticInferenceServiceAuthorizationModelTests.createEnabledAuth(), gatewayUrl); } private ElasticInferenceService createService( HttpRequestSender.Factory senderFactory, - ElasticInferenceServiceAuthorization auth, + ElasticInferenceServiceAuthorizationModel auth, String gatewayUrl ) { - var mockAuthHandler = mock(ElasticInferenceServiceAuthorizationHandler.class); + var mockAuthHandler = mock(ElasticInferenceServiceAuthorizationRequestHandler.class); doAnswer(invocation -> { - ActionListener listener = invocation.getArgument(0); + ActionListener listener = invocation.getArgument(0); listener.onResponse(auth); return Void.TYPE; }).when(mockAuthHandler).getAuthorization(any(), any()); @@ -994,7 +1127,7 @@ private ElasticInferenceService createService( return new ElasticInferenceService( senderFactory, createWithEmptySettings(threadPool), - new ElasticInferenceServiceComponents(gatewayUrl), + ElasticInferenceServiceSettingsTests.create(gatewayUrl), mockModelRegistry(), mockAuthHandler ); @@ -1004,9 +1137,23 @@ private ElasticInferenceService createServiceWithAuthHandler(HttpRequestSender.F return new ElasticInferenceService( senderFactory, createWithEmptySettings(threadPool), - new ElasticInferenceServiceComponents(eisGatewayUrl), + ElasticInferenceServiceSettingsTests.create(eisGatewayUrl), mockModelRegistry(), - new ElasticInferenceServiceAuthorizationHandler(eisGatewayUrl, threadPool) + new ElasticInferenceServiceAuthorizationRequestHandler(eisGatewayUrl, threadPool) + ); + } + + public static ElasticInferenceService createServiceWithAuthHandler( + HttpRequestSender.Factory senderFactory, + String eisGatewayUrl, + ThreadPool threadPool + ) { + return new ElasticInferenceService( + senderFactory, + createWithEmptySettings(threadPool), + ElasticInferenceServiceSettingsTests.create(eisGatewayUrl), + mockModelRegistry(threadPool), + new ElasticInferenceServiceAuthorizationRequestHandler(eisGatewayUrl, threadPool) ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandlerTests.java index a819bf1b4a513..5435d5b9a6dad 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandlerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationHandlerTests.java @@ -7,264 +7,165 @@ package org.elasticsearch.xpack.inference.services.elastic.authorization; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.EmptySecretSettings; +import org.elasticsearch.inference.EmptyTaskSettings; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.MinimalServiceSettings; +import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.http.MockResponse; -import org.elasticsearch.test.http.MockWebServer; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; -import org.elasticsearch.xpack.inference.external.http.HttpClientManager; -import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; -import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; -import org.junit.After; +import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceAuthorizationResponseEntity; +import org.elasticsearch.xpack.inference.services.elastic.DefaultModelConfig; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceComponents; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionModel; +import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionServiceSettings; import org.junit.Before; -import org.mockito.ArgumentCaptor; import java.io.IOException; import java.util.EnumSet; import java.util.List; -import java.util.Set; +import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; -import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; -import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; -import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; -import static org.elasticsearch.xpack.inference.external.http.retry.RetryingHttpSender.MAX_RETIES; -import static org.hamcrest.Matchers.is; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService.defaultEndpointId; +import static org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceTests.mockModelRegistry; +import static org.hamcrest.CoreMatchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; public class ElasticInferenceServiceAuthorizationHandlerTests extends ESTestCase { - private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); - private final MockWebServer webServer = new MockWebServer(); - private ThreadPool threadPool; - - private HttpClientManager clientManager; + private DeterministicTaskQueue taskQueue; @Before public void init() throws Exception { - webServer.start(); - threadPool = createThreadPool(inferenceUtilityPool()); - clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); - } - - @After - public void shutdown() throws IOException { - clientManager.close(); - terminate(threadPool); - webServer.close(); - } - - public void testDoesNotAttempt_ToRetrieveAuthorization_IfBaseUrlIsNull() throws Exception { - var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); - var logger = mock(Logger.class); - var authHandler = new ElasticInferenceServiceAuthorizationHandler(null, threadPool, logger); - - try (var sender = senderFactory.createSender()) { - PlainActionFuture listener = new PlainActionFuture<>(); - authHandler.getAuthorization(listener, sender); - - var authResponse = listener.actionGet(TIMEOUT); - assertTrue(authResponse.getAuthorizedTaskTypes().isEmpty()); - assertTrue(authResponse.getAuthorizedModelIds().isEmpty()); - assertFalse(authResponse.isAuthorized()); - - var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); - verify(logger).warn(loggerArgsCaptor.capture()); - var message = loggerArgsCaptor.getValue(); - assertThat(message, is("The base URL for the authorization service is not valid, rejecting authorization.")); - } + taskQueue = new DeterministicTaskQueue(); } - public void testDoesNotAttempt_ToRetrieveAuthorization_IfBaseUrlIsEmpty() throws Exception { - var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); - var logger = mock(Logger.class); - var authHandler = new ElasticInferenceServiceAuthorizationHandler("", threadPool, logger); - - try (var sender = senderFactory.createSender()) { - PlainActionFuture listener = new PlainActionFuture<>(); - authHandler.getAuthorization(listener, sender); - - var authResponse = listener.actionGet(TIMEOUT); - assertTrue(authResponse.getAuthorizedTaskTypes().isEmpty()); - assertTrue(authResponse.getAuthorizedModelIds().isEmpty()); - assertFalse(authResponse.isAuthorized()); - - var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); - verify(logger).warn(loggerArgsCaptor.capture()); - var message = loggerArgsCaptor.getValue(); - assertThat(message, is("The base URL for the authorization service is not valid, rejecting authorization.")); - } - } - - public void testGetAuthorization_FailsWhenAnInvalidFieldIsFound() throws IOException { - var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); - var eisGatewayUrl = getUrl(webServer); - var logger = mock(Logger.class); - var authHandler = new ElasticInferenceServiceAuthorizationHandler(eisGatewayUrl, threadPool, logger); - - try (var sender = senderFactory.createSender()) { - String responseJson = """ - { - "models": [ - { - "invalid-field": "model-a", - "task-types": ["embed/text/sparse", "chat"] - } - ] + public void testSendsAnAuthorizationRequestTwice() throws Exception { + var callbackCount = new AtomicInteger(0); + // we're only interested in two authorization calls which is why I'm using a value of 2 here + var latch = new CountDownLatch(2); + final AtomicReference handlerRef = new AtomicReference<>(); + + Runnable callback = () -> { + // the first authorization response does not contain a streaming task so we're expecting to not support streaming here + if (callbackCount.incrementAndGet() == 1) { + assertThat(handlerRef.get().supportedStreamingTasks(), is(EnumSet.noneOf(TaskType.class))); + } + latch.countDown(); + + // we only want to run the tasks twice, so advance the time on the queue + // which flags the scheduled authorization request to be ready to run + if (callbackCount.get() == 1) { + taskQueue.advanceTime(); + } else { + try { + handlerRef.get().close(); + } catch (IOException e) { + // ignore } - """; - - queueWebServerResponsesForRetries(responseJson); - - PlainActionFuture listener = new PlainActionFuture<>(); - authHandler.getAuthorization(listener, sender); - - var authResponse = listener.actionGet(TIMEOUT); - assertTrue(authResponse.getAuthorizedTaskTypes().isEmpty()); - assertTrue(authResponse.getAuthorizedModelIds().isEmpty()); - assertFalse(authResponse.isAuthorized()); - - var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); - verify(logger).warn(loggerArgsCaptor.capture()); - var message = loggerArgsCaptor.getValue(); - assertThat( - message, - is( - "Failed to retrieve the authorization information from the Elastic Inference Service." - + " Encountered an exception: org.elasticsearch.xcontent.XContentParseException: [4:28] " - + "[ElasticInferenceServiceAuthorizationResponseEntity] failed to parse field [models]" + } + }; + + var requestHandler = mockAuthorizationRequestHandler( + ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel("abc", EnumSet.of(TaskType.SPARSE_EMBEDDING)) + ) ) - ); - } - } - - /** - * Queues the required number of responses to handle the retries of the internal sender. - */ - private void queueWebServerResponsesForRetries(String responseJson) { - for (int i = 0; i < MAX_RETIES; i++) { - webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - } - } - - public void testGetAuthorization_ReturnsAValidResponse() throws IOException { - var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); - var eisGatewayUrl = getUrl(webServer); - var logger = mock(Logger.class); - var authHandler = new ElasticInferenceServiceAuthorizationHandler(eisGatewayUrl, threadPool, logger); - - try (var sender = senderFactory.createSender()) { - String responseJson = """ - { - "models": [ - { - "model_name": "model-a", - "task_types": ["embed/text/sparse", "chat"] - } - ] - } - """; - - webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - - PlainActionFuture listener = new PlainActionFuture<>(); - authHandler.getAuthorization(listener, sender); - - var authResponse = listener.actionGet(TIMEOUT); - assertThat(authResponse.getAuthorizedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION))); - assertThat(authResponse.getAuthorizedModelIds(), is(Set.of("model-a"))); - assertTrue(authResponse.isAuthorized()); - - var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); - verify(logger, times(1)).debug(loggerArgsCaptor.capture()); - - var message = loggerArgsCaptor.getValue(); - assertThat(message, is("Retrieving authorization information from the Elastic Inference Service.")); - verifyNoMoreInteractions(logger); - } - } - - @SuppressWarnings("unchecked") - public void testGetAuthorization_OnResponseCalledOnce() throws IOException { - var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); - var eisGatewayUrl = getUrl(webServer); - var logger = mock(Logger.class); - var authHandler = new ElasticInferenceServiceAuthorizationHandler(eisGatewayUrl, threadPool, logger); - - ActionListener listener = mock(ActionListener.class); - String responseJson = """ - { - "models": [ - { - "model_name": "model-a", - "task_types": ["embed/text/sparse", "chat"] - } - ] - } - """; - webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - - try (var sender = senderFactory.createSender()) { - authHandler.getAuthorization(listener, sender); - authHandler.waitForAuthRequestCompletion(TIMEOUT); - - verify(listener, times(1)).onResponse(any()); - var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); - verify(logger, times(1)).debug(loggerArgsCaptor.capture()); - - var message = loggerArgsCaptor.getValue(); - assertThat(message, is("Retrieving authorization information from the Elastic Inference Service.")); - verifyNoMoreInteractions(logger); - } + ), + ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "rainbow-sprinkles", + EnumSet.of(TaskType.CHAT_COMPLETION) + ) + ) + ) + ) + ); + + handlerRef.set( + new ElasticInferenceServiceAuthorizationHandler( + createWithEmptySettings(taskQueue.getThreadPool()), + mockModelRegistry(taskQueue.getThreadPool()), + requestHandler, + initDefaultEndpoints(), + EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION), + null, + mock(Sender.class), + ElasticInferenceServiceSettingsTests.create(null, TimeValue.timeValueMillis(1), TimeValue.timeValueMillis(1), true), + callback + ) + ); + + var handler = handlerRef.get(); + handler.init(); + taskQueue.runAllRunnableTasks(); + latch.await(Utils.TIMEOUT.getSeconds(), TimeUnit.SECONDS); + // this should be after we've received both authorization responses + + assertThat(handler.supportedStreamingTasks(), is(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.ANY))); + assertThat( + handler.defaultConfigIds(), + is(List.of(new InferenceService.DefaultConfigId(".rainbow-sprinkles-elastic", MinimalServiceSettings.chatCompletion(), null))) + ); + assertThat(handler.supportedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION))); + + PlainActionFuture> listener = new PlainActionFuture<>(); + handler.defaultConfigs(listener); + + var configs = listener.actionGet(); + assertThat(configs.get(0).getConfigurations().getInferenceEntityId(), is(".rainbow-sprinkles-elastic")); } - public void testGetAuthorization_InvalidResponse() throws IOException { - var senderMock = mock(Sender.class); - var senderFactory = mock(HttpRequestSender.Factory.class); - when(senderFactory.createSender()).thenReturn(senderMock); - - doAnswer(invocationOnMock -> { - ActionListener listener = invocationOnMock.getArgument(4); - listener.onResponse(new ChatCompletionResults(List.of(new ChatCompletionResults.Result("awesome")))); + private static ElasticInferenceServiceAuthorizationRequestHandler mockAuthorizationRequestHandler( + ElasticInferenceServiceAuthorizationModel firstAuthResponse, + ElasticInferenceServiceAuthorizationModel secondAuthResponse + ) { + var mockAuthHandler = mock(ElasticInferenceServiceAuthorizationRequestHandler.class); + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(firstAuthResponse); return Void.TYPE; - }).when(senderMock).sendWithoutQueuing(any(), any(), any(), any(), any()); - - var logger = mock(Logger.class); - var authHandler = new ElasticInferenceServiceAuthorizationHandler("abc", threadPool, logger); - - try (var sender = senderFactory.createSender()) { - PlainActionFuture listener = new PlainActionFuture<>(); - - authHandler.getAuthorization(listener, sender); - var result = listener.actionGet(TIMEOUT); - - assertThat(result, is(ElasticInferenceServiceAuthorization.newDisabledService())); + }).doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(secondAuthResponse); + return Void.TYPE; + }).when(mockAuthHandler).getAuthorization(any(), any()); - var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); - verify(logger).warn(loggerArgsCaptor.capture()); - var message = loggerArgsCaptor.getValue(); - assertThat( - message, - is( - "Failed to retrieve the authorization information from the Elastic Inference Service." - + " Received an invalid response type: ChatCompletionResults" - ) - ); - } + return mockAuthHandler; + } + private static Map initDefaultEndpoints() { + return Map.of( + "rainbow-sprinkles", + new DefaultModelConfig( + new ElasticInferenceServiceCompletionModel( + defaultEndpointId("rainbow-sprinkles"), + TaskType.CHAT_COMPLETION, + "test", + new ElasticInferenceServiceCompletionServiceSettings("rainbow-sprinkles", null), + EmptyTaskSettings.INSTANCE, + EmptySecretSettings.INSTANCE, + ElasticInferenceServiceComponents.EMPTY_INSTANCE + ), + MinimalServiceSettings.chatCompletion() + ) + ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationModelTests.java similarity index 53% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationModelTests.java index 559de47232a7b..6db9238ab65a4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationModelTests.java @@ -17,9 +17,9 @@ import static org.hamcrest.Matchers.is; -public class ElasticInferenceServiceAuthorizationTests extends ESTestCase { - public static ElasticInferenceServiceAuthorization createEnabledAuth() { - return ElasticInferenceServiceAuthorization.of( +public class ElasticInferenceServiceAuthorizationModelTests extends ESTestCase { + public static ElasticInferenceServiceAuthorizationModel createEnabledAuth() { + return ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel("model-1", EnumSet.of(TaskType.TEXT_EMBEDDING)) @@ -29,20 +29,20 @@ public static ElasticInferenceServiceAuthorization createEnabledAuth() { } public void testIsAuthorized_ReturnsFalse_WithEmptyMap() { - assertFalse(ElasticInferenceServiceAuthorization.newDisabledService().isAuthorized()); + assertFalse(ElasticInferenceServiceAuthorizationModel.newDisabledService().isAuthorized()); } public void testExcludes_ModelsWithoutTaskTypes() { var response = new ElasticInferenceServiceAuthorizationResponseEntity( List.of(new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel("model-1", EnumSet.noneOf(TaskType.class))) ); - var auth = ElasticInferenceServiceAuthorization.of(response); + var auth = ElasticInferenceServiceAuthorizationModel.of(response); assertTrue(auth.getAuthorizedTaskTypes().isEmpty()); assertFalse(auth.isAuthorized()); } public void testEnabledTaskTypes_MergesFromSeparateModels() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel("model-1", EnumSet.of(TaskType.TEXT_EMBEDDING)), @@ -55,7 +55,7 @@ public void testEnabledTaskTypes_MergesFromSeparateModels() { } public void testEnabledTaskTypes_FromSingleEntry() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -71,7 +71,7 @@ public void testEnabledTaskTypes_FromSingleEntry() { } public void testNewLimitToTaskTypes_SingleModel() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -86,7 +86,7 @@ public void testNewLimitToTaskTypes_SingleModel() { assertThat( auth.newLimitedToTaskTypes(EnumSet.of(TaskType.TEXT_EMBEDDING)), is( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -101,7 +101,7 @@ public void testNewLimitToTaskTypes_SingleModel() { } public void testNewLimitToTaskTypes_MultipleModels_OnlyTextEmbedding() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -116,7 +116,7 @@ public void testNewLimitToTaskTypes_MultipleModels_OnlyTextEmbedding() { assertThat( auth.newLimitedToTaskTypes(EnumSet.of(TaskType.TEXT_EMBEDDING)), is( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -135,7 +135,7 @@ public void testNewLimitToTaskTypes_MultipleModels_OnlyTextEmbedding() { } public void testNewLimitToTaskTypes_MultipleModels_MultipleTaskTypes() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -154,11 +154,11 @@ public void testNewLimitToTaskTypes_MultipleModels_MultipleTaskTypes() { ) ); - var a = auth.newLimitedToTaskTypes(EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.CHAT_COMPLETION)); + var limitedAuth = auth.newLimitedToTaskTypes(EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.CHAT_COMPLETION)); assertThat( - a, + limitedAuth, is( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -177,7 +177,7 @@ public void testNewLimitToTaskTypes_MultipleModels_MultipleTaskTypes() { } public void testNewLimitToTaskTypes_DuplicateModelNames() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -192,11 +192,11 @@ public void testNewLimitToTaskTypes_DuplicateModelNames() { ) ); - var a = auth.newLimitedToTaskTypes(EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING, TaskType.RERANK)); + var limitedAuth = auth.newLimitedToTaskTypes(EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING, TaskType.RERANK)); assertThat( - a, + limitedAuth, is( - ElasticInferenceServiceAuthorization.of( + ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -211,7 +211,7 @@ public void testNewLimitToTaskTypes_DuplicateModelNames() { } public void testNewLimitToTaskTypes_ReturnsDisabled_WhenNoOverlapForTaskTypes() { - var auth = ElasticInferenceServiceAuthorization.of( + var auth = ElasticInferenceServiceAuthorizationModel.of( new ElasticInferenceServiceAuthorizationResponseEntity( List.of( new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( @@ -226,7 +226,160 @@ public void testNewLimitToTaskTypes_ReturnsDisabled_WhenNoOverlapForTaskTypes() ) ); - var a = auth.newLimitedToTaskTypes(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.RERANK)); - assertThat(a, is(ElasticInferenceServiceAuthorization.newDisabledService())); + var limitedAuth = auth.newLimitedToTaskTypes(EnumSet.of(TaskType.CHAT_COMPLETION, TaskType.RERANK)); + assertThat(limitedAuth, is(ElasticInferenceServiceAuthorizationModel.newDisabledService())); + } + + public void testMerge_CombinesCorrectly() { + var auth1 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ); + + var auth2 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel("model-2", EnumSet.of(TaskType.SPARSE_EMBEDDING)) + ) + ) + ); + + assertThat( + auth1.merge(auth2), + is( + ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ), + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-2", + EnumSet.of(TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ) + ) + ); + } + + public void testMerge_AddsNewTaskType() { + var auth1 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ); + + var auth2 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel("model-2", EnumSet.of(TaskType.CHAT_COMPLETION)) + ) + ) + ); + + assertThat( + auth1.merge(auth2), + is( + ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ), + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-2", + EnumSet.of(TaskType.CHAT_COMPLETION) + ) + ) + ) + ) + ) + ); + } + + public void testMerge_IgnoresDuplicates() { + var auth1 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ); + + var auth2 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ); + + assertThat( + auth1.merge(auth2), + is( + ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ) + ) + ); + } + + public void testMerge_CombinesCorrectlyWithEmptyModel() { + var auth1 = ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ); + + var auth2 = ElasticInferenceServiceAuthorizationModel.newDisabledService(); + + assertThat( + auth1.merge(auth2), + is( + ElasticInferenceServiceAuthorizationModel.of( + new ElasticInferenceServiceAuthorizationResponseEntity( + List.of( + new ElasticInferenceServiceAuthorizationResponseEntity.AuthorizedModel( + "model-1", + EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING) + ) + ) + ) + ) + ) + ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationRequestHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationRequestHandlerTests.java new file mode 100644 index 0000000000000..380c0e8b3be94 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/authorization/ElasticInferenceServiceAuthorizationRequestHandlerTests.java @@ -0,0 +1,272 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic.authorization; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.junit.After; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.external.http.retry.RetryingHttpSender.MAX_RETIES; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class ElasticInferenceServiceAuthorizationRequestHandlerTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testDoesNotAttempt_ToRetrieveAuthorization_IfBaseUrlIsNull() throws Exception { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var logger = mock(Logger.class); + var authHandler = new ElasticInferenceServiceAuthorizationRequestHandler(null, threadPool, logger); + + try (var sender = senderFactory.createSender()) { + PlainActionFuture listener = new PlainActionFuture<>(); + authHandler.getAuthorization(listener, sender); + + var authResponse = listener.actionGet(TIMEOUT); + assertTrue(authResponse.getAuthorizedTaskTypes().isEmpty()); + assertTrue(authResponse.getAuthorizedModelIds().isEmpty()); + assertFalse(authResponse.isAuthorized()); + + var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); + verify(logger, times(2)).debug(loggerArgsCaptor.capture()); + var messages = loggerArgsCaptor.getAllValues(); + assertThat(messages.get(0), is("Retrieving authorization information from the Elastic Inference Service.")); + assertThat(messages.get(1), is("The base URL for the authorization service is not valid, rejecting authorization.")); + } + } + + public void testDoesNotAttempt_ToRetrieveAuthorization_IfBaseUrlIsEmpty() throws Exception { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var logger = mock(Logger.class); + var authHandler = new ElasticInferenceServiceAuthorizationRequestHandler("", threadPool, logger); + + try (var sender = senderFactory.createSender()) { + PlainActionFuture listener = new PlainActionFuture<>(); + authHandler.getAuthorization(listener, sender); + + var authResponse = listener.actionGet(TIMEOUT); + assertTrue(authResponse.getAuthorizedTaskTypes().isEmpty()); + assertTrue(authResponse.getAuthorizedModelIds().isEmpty()); + assertFalse(authResponse.isAuthorized()); + + var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); + verify(logger, times(2)).debug(loggerArgsCaptor.capture()); + var messages = loggerArgsCaptor.getAllValues(); + assertThat(messages.get(0), is("Retrieving authorization information from the Elastic Inference Service.")); + assertThat(messages.get(1), is("The base URL for the authorization service is not valid, rejecting authorization.")); + } + } + + public void testGetAuthorization_FailsWhenAnInvalidFieldIsFound() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var eisGatewayUrl = getUrl(webServer); + var logger = mock(Logger.class); + var authHandler = new ElasticInferenceServiceAuthorizationRequestHandler(eisGatewayUrl, threadPool, logger); + + try (var sender = senderFactory.createSender()) { + String responseJson = """ + { + "models": [ + { + "invalid-field": "model-a", + "task-types": ["embed/text/sparse", "chat"] + } + ] + } + """; + + queueWebServerResponsesForRetries(responseJson); + + PlainActionFuture listener = new PlainActionFuture<>(); + authHandler.getAuthorization(listener, sender); + + var authResponse = listener.actionGet(TIMEOUT); + assertTrue(authResponse.getAuthorizedTaskTypes().isEmpty()); + assertTrue(authResponse.getAuthorizedModelIds().isEmpty()); + assertFalse(authResponse.isAuthorized()); + + var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); + verify(logger).warn(loggerArgsCaptor.capture()); + var message = loggerArgsCaptor.getValue(); + assertThat( + message, + is( + "Failed to retrieve the authorization information from the Elastic Inference Service." + + " Encountered an exception: org.elasticsearch.xcontent.XContentParseException: [4:28] " + + "[ElasticInferenceServiceAuthorizationResponseEntity] failed to parse field [models]" + ) + ); + } + } + + /** + * Queues the required number of responses to handle the retries of the internal sender. + */ + private void queueWebServerResponsesForRetries(String responseJson) { + for (int i = 0; i < MAX_RETIES; i++) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + } + } + + public void testGetAuthorization_ReturnsAValidResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var eisGatewayUrl = getUrl(webServer); + var logger = mock(Logger.class); + var authHandler = new ElasticInferenceServiceAuthorizationRequestHandler(eisGatewayUrl, threadPool, logger); + + try (var sender = senderFactory.createSender()) { + String responseJson = """ + { + "models": [ + { + "model_name": "model-a", + "task_types": ["embed/text/sparse", "chat"] + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + PlainActionFuture listener = new PlainActionFuture<>(); + authHandler.getAuthorization(listener, sender); + + var authResponse = listener.actionGet(TIMEOUT); + assertThat(authResponse.getAuthorizedTaskTypes(), is(EnumSet.of(TaskType.SPARSE_EMBEDDING, TaskType.CHAT_COMPLETION))); + assertThat(authResponse.getAuthorizedModelIds(), is(Set.of("model-a"))); + assertTrue(authResponse.isAuthorized()); + + var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); + verify(logger, times(1)).debug(loggerArgsCaptor.capture()); + + var message = loggerArgsCaptor.getValue(); + assertThat(message, is("Retrieving authorization information from the Elastic Inference Service.")); + verifyNoMoreInteractions(logger); + } + } + + @SuppressWarnings("unchecked") + public void testGetAuthorization_OnResponseCalledOnce() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var eisGatewayUrl = getUrl(webServer); + var logger = mock(Logger.class); + var authHandler = new ElasticInferenceServiceAuthorizationRequestHandler(eisGatewayUrl, threadPool, logger); + + ActionListener listener = mock(ActionListener.class); + String responseJson = """ + { + "models": [ + { + "model_name": "model-a", + "task_types": ["embed/text/sparse", "chat"] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + try (var sender = senderFactory.createSender()) { + authHandler.getAuthorization(listener, sender); + authHandler.waitForAuthRequestCompletion(TIMEOUT); + + verify(listener, times(1)).onResponse(any()); + var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); + verify(logger, times(1)).debug(loggerArgsCaptor.capture()); + + var message = loggerArgsCaptor.getValue(); + assertThat(message, is("Retrieving authorization information from the Elastic Inference Service.")); + verifyNoMoreInteractions(logger); + } + } + + public void testGetAuthorization_InvalidResponse() throws IOException { + var senderMock = mock(Sender.class); + var senderFactory = mock(HttpRequestSender.Factory.class); + when(senderFactory.createSender()).thenReturn(senderMock); + + doAnswer(invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(4); + listener.onResponse(new ChatCompletionResults(List.of(new ChatCompletionResults.Result("awesome")))); + return Void.TYPE; + }).when(senderMock).sendWithoutQueuing(any(), any(), any(), any(), any()); + + var logger = mock(Logger.class); + var authHandler = new ElasticInferenceServiceAuthorizationRequestHandler("abc", threadPool, logger); + + try (var sender = senderFactory.createSender()) { + PlainActionFuture listener = new PlainActionFuture<>(); + + authHandler.getAuthorization(listener, sender); + var result = listener.actionGet(TIMEOUT); + + assertThat(result, is(ElasticInferenceServiceAuthorizationModel.newDisabledService())); + + var loggerArgsCaptor = ArgumentCaptor.forClass(String.class); + verify(logger).warn(loggerArgsCaptor.capture()); + var message = loggerArgsCaptor.getValue(); + assertThat( + message, + is( + "Failed to retrieve the authorization information from the Elastic Inference Service." + + " Received an invalid response type: ChatCompletionResults" + ) + ); + } + + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java index 07da96cb32273..51945776b4f9e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/completion/ElasticInferenceServiceCompletionModelTests.java @@ -29,7 +29,7 @@ public void testOverridingModelId() { new ElasticInferenceServiceCompletionServiceSettings("model_id", new RateLimitSettings(100)), EmptyTaskSettings.INSTANCE, EmptySecretSettings.INSTANCE, - new ElasticInferenceServiceComponents("url") + ElasticInferenceServiceComponents.of("url") ); var request = new UnifiedCompletionRequest( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java index ebb9c964e4c9a..c9eccd02b771b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceFields; @@ -31,22 +32,23 @@ public class CustomElandInternalTextEmbeddingServiceSettingsTests extends Abstra CustomElandInternalTextEmbeddingServiceSettings> { public static CustomElandInternalTextEmbeddingServiceSettings createRandom() { - var numAllocations = randomIntBetween(1, 10); + var withAdaptiveAllocations = randomBoolean(); + var numAllocations = withAdaptiveAllocations ? null : randomIntBetween(1, 10); + var adaptiveAllocationsSettings = withAdaptiveAllocations + ? new AdaptiveAllocationsSettings(true, randomIntBetween(0, 2), randomIntBetween(2, 5)) + : null; var numThreads = randomIntBetween(1, 10); var modelId = randomAlphaOfLength(8); - SimilarityMeasure similarityMeasure = SimilarityMeasure.COSINE; - Integer dims = null; + var similarityMeasure = SimilarityMeasure.COSINE; var setDimensions = randomBoolean(); - if (setDimensions) { - dims = 123; - } - + var dims = setDimensions ? 123 : null; var elementType = randomFrom(DenseVectorFieldMapper.ElementType.values()); return new CustomElandInternalTextEmbeddingServiceSettings( numAllocations, numThreads, modelId, + adaptiveAllocationsSettings, null, dims, similarityMeasure, @@ -86,6 +88,7 @@ public void testFromMap_Request_CreatesSettingsCorrectly() { modelId, null, null, + null, SimilarityMeasure.DOT_PRODUCT, DenseVectorFieldMapper.ElementType.FLOAT ) @@ -111,6 +114,7 @@ public void testFromMap_Request_DoesNotDefaultSimilarityElementType() { modelId, null, null, + null, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT ) @@ -152,6 +156,7 @@ public void testFromMap_Request_IgnoresDimensions() { modelId, null, null, + null, SimilarityMeasure.DOT_PRODUCT, DenseVectorFieldMapper.ElementType.FLOAT ) @@ -192,6 +197,7 @@ public void testFromMap_Persistent_CreatesSettingsCorrectly() { numThreads, modelId, null, + null, 1, SimilarityMeasure.DOT_PRODUCT, DenseVectorFieldMapper.ElementType.FLOAT @@ -206,6 +212,7 @@ public void testToXContent_WritesAllValues() throws IOException { 1, "model_id", null, + null, 100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.BYTE diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java index 0db0a7669c8aa..4ec575420613f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java @@ -103,7 +103,7 @@ public void testFromMap() { ) ) ).build(); - assertEquals(new ElasticsearchInternalServiceSettings(1, 4, ".elser_model_1", null), serviceSettings); + assertEquals(new ElasticsearchInternalServiceSettings(1, 4, ".elser_model_1", null, null), serviceSettings); } public void testFromMapMissingOptions() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index 3b634f45dc751..d8886e1eea471 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; @@ -46,12 +47,14 @@ import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; import org.elasticsearch.xpack.core.ml.MachineLearningField; +import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; +import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResultsTests; @@ -67,13 +70,16 @@ import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -81,12 +87,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction.Response.RESULTS_FIELD; import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86; @@ -101,6 +109,8 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; public class ElasticsearchInternalServiceTests extends ESTestCase { @@ -351,7 +361,9 @@ public void testParseRequestConfig_elser() { ) ); - var elserServiceSettings = new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null); + var elserServiceSettings = new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null, null) + ); service.parseRequestConfig( randomInferenceEntityId, @@ -381,7 +393,9 @@ public void testParseRequestConfig_elser() { ) ); - var elserServiceSettings = new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null); + var elserServiceSettings = new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null, null) + ); String criticalWarning = "Putting elasticsearch service inference endpoints (including elser service) without a model_id field is" @@ -450,7 +464,9 @@ public void testParseRequestConfig_elser() { ); config.put(ModelConfigurations.CHUNKING_SETTINGS, createRandomChunkingSettingsMap()); - var elserServiceSettings = new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null); + var elserServiceSettings = new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null, null) + ); service.parseRequestConfig( randomInferenceEntityId, @@ -486,7 +502,9 @@ public void testParseRequestConfig_elser() { ) ); - var elserServiceSettings = new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null); + var elserServiceSettings = new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null, null) + ); service.parseRequestConfig( randomInferenceEntityId, @@ -692,6 +710,30 @@ private ActionListener getElserModelVerificationActionListener( public void testParsePersistedConfig() { + // Parsing a persistent configuration using model_version succeeds + { + var service = createService(mock(Client.class)); + var settings = new HashMap(); + settings.put( + ModelConfigurations.SERVICE_SETTINGS, + new HashMap<>( + Map.of( + ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, + 1, + ElasticsearchInternalServiceSettings.NUM_THREADS, + 4, + "model_version", + ".elser_model_2" + ) + ) + ); + + var model = service.parsePersistedConfig(randomInferenceEntityId, TaskType.TEXT_EMBEDDING, settings); + assertThat(model, instanceOf(ElserInternalModel.class)); + ElserInternalModel elserInternalModel = (ElserInternalModel) model; + assertThat(elserInternalModel.getServiceSettings().modelId(), is(".elser_model_2")); + } + // Null model variant { var service = createService(mock(Client.class)); @@ -710,11 +752,12 @@ public void testParsePersistedConfig() { ) ); - expectThrows( + var exception = expectThrows( IllegalArgumentException.class, () -> service.parsePersistedConfig(randomInferenceEntityId, TaskType.TEXT_EMBEDDING, settings) ); + assertThat(exception.getMessage(), containsString(randomInferenceEntityId)); } // Invalid model variant @@ -742,7 +785,16 @@ public void testParsePersistedConfig() { TaskType.TEXT_EMBEDDING, settings ); - var elandServiceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "invalid", null); + var elandServiceSettings = new CustomElandInternalTextEmbeddingServiceSettings( + 1, + 4, + "invalid", + null, + null, + null, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); assertEquals( new CustomElandEmbeddingModel( randomInferenceEntityId, @@ -933,7 +985,7 @@ private void testChunkInfer_Sparse(ChunkingSettings chunkingSettings) throws Int "foo", TaskType.SPARSE_EMBEDDING, "elasticsearch", - new ElasticsearchInternalServiceSettings(1, 1, "model-id", null), + new ElasticsearchInternalServiceSettings(1, 1, "model-id", null, null), chunkingSettings ); var service = createService(client); @@ -1003,7 +1055,7 @@ private void testChunkInfer_Elser(ChunkingSettings chunkingSettings) throws Inte "foo", TaskType.SPARSE_EMBEDDING, "elasticsearch", - new ElserInternalServiceSettings(1, 1, "model-id", null), + new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, "model-id", null, null)), new ElserMlNodeTaskSettings(), chunkingSettings ); @@ -1328,11 +1380,20 @@ private CustomElandModel getCustomElandModel(TaskType taskType) { randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, - new CustomElandInternalServiceSettings(1, 4, "custom-model", null), + new CustomElandInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 4, "custom-model", null, null)), RerankTaskSettings.DEFAULT_SETTINGS ); } else if (taskType == TaskType.TEXT_EMBEDDING) { - var serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "custom-model", null); + var serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings( + 1, + 4, + "custom-model", + null, + null, + null, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); expectedModel = new CustomElandEmbeddingModel( randomInferenceEntityId, @@ -1346,7 +1407,7 @@ private CustomElandModel getCustomElandModel(TaskType taskType) { randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, - new CustomElandInternalServiceSettings(1, 4, "custom-model", null), + new CustomElandInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 4, "custom-model", null, null)), (ChunkingSettings) null ); } @@ -1438,6 +1499,7 @@ public void testParseRequestConfigEland_SetsDimensionsToOne() { 4, "custom-model", null, + null, 1, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT @@ -1463,6 +1525,7 @@ public void testParseRequestConfigEland_SetsDimensionsToOne() { "custom-model", null, null, + null, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT ), @@ -1511,7 +1574,7 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() { EmbeddingRequestChunker.EmbeddingType.SPARSE, ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( TaskType.SPARSE_EMBEDDING, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + new ElasticsearchInternalServiceSettings(1, 1, "foo", null, null) ) ); assertEquals( @@ -1526,7 +1589,7 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() { ElasticsearchStatusException.class, () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( TaskType.COMPLETION, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + new ElasticsearchInternalServiceSettings(1, 1, "foo", null, null) ) ); assertThat(e1.getMessage(), containsString("Chunking is not supported for task type [completion]")); @@ -1535,7 +1598,7 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() { ElasticsearchStatusException.class, () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( TaskType.RERANK, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + new ElasticsearchInternalServiceSettings(1, 1, "foo", null, null) ) ); assertThat(e2.getMessage(), containsString("Chunking is not supported for task type [rerank]")); @@ -1604,6 +1667,209 @@ public void testGetConfiguration() throws Exception { } } + public void testUpdateModelsWithDynamicFields_NoModelsToUpdate() throws Exception { + ActionListener> resultsListener = ActionListener.>wrap( + updatedModels -> assertEquals(Collections.emptyList(), updatedModels), + e -> fail("Unexpected exception: " + e) + ); + + try (var service = createService(mock(Client.class))) { + service.updateModelsWithDynamicFields(List.of(), resultsListener); + } + } + + public void testUpdateModelsWithDynamicFields_InvalidModelProvided() throws IOException { + ActionListener> resultsListener = ActionListener.wrap( + updatedModels -> fail("Expected invalid model assertion error to be thrown"), + e -> fail("Expected invalid model assertion error to be thrown") + ); + + try (var service = createService(mock(Client.class))) { + assertThrows( + AssertionError.class, + () -> { service.updateModelsWithDynamicFields(List.of(mock(Model.class)), resultsListener); } + ); + } + } + + @SuppressWarnings("unchecked") + public void testUpdateModelsWithDynamicFields_FailsToRetrieveDeployments() throws IOException { + var deploymentId = randomAlphaOfLength(10); + var model = mock(ElasticsearchInternalModel.class); + when(model.mlNodeDeploymentId()).thenReturn(deploymentId); + when(model.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); + + ActionListener> resultsListener = ActionListener.wrap(updatedModels -> { + assertEquals(updatedModels.size(), 1); + verify(model).mlNodeDeploymentId(); + verifyNoMoreInteractions(model); + }, e -> fail("Expected original models to be returned")); + + var client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocation -> { + var listener = (ActionListener) invocation.getArguments()[2]; + listener.onFailure(new RuntimeException(randomAlphaOfLength(10))); + return null; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + + try (var service = createService(client)) { + service.updateModelsWithDynamicFields(List.of(model), resultsListener); + } + } + + public void testUpdateModelsWithDynamicFields_SingleModelToUpdate() throws IOException { + var deploymentId = randomAlphaOfLength(10); + var model = mock(ElasticsearchInternalModel.class); + when(model.mlNodeDeploymentId()).thenReturn(deploymentId); + when(model.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); + + var modelsByDeploymentId = new HashMap>(); + modelsByDeploymentId.put(deploymentId, List.of(model)); + + testUpdateModelsWithDynamicFields(modelsByDeploymentId); + } + + public void testUpdateModelsWithDynamicFields_MultipleModelsWithDifferentDeploymentsToUpdate() throws IOException { + var deploymentId1 = randomAlphaOfLength(10); + var model1 = mock(ElasticsearchInternalModel.class); + when(model1.mlNodeDeploymentId()).thenReturn(deploymentId1); + when(model1.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); + var deploymentId2 = randomAlphaOfLength(10); + var model2 = mock(ElasticsearchInternalModel.class); + when(model2.mlNodeDeploymentId()).thenReturn(deploymentId2); + when(model2.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); + + var modelsByDeploymentId = new HashMap>(); + modelsByDeploymentId.put(deploymentId1, List.of(model1)); + modelsByDeploymentId.put(deploymentId2, List.of(model2)); + + testUpdateModelsWithDynamicFields(modelsByDeploymentId); + } + + public void testUpdateModelsWithDynamicFields_MultipleModelsWithSameDeploymentsToUpdate() throws IOException { + var deploymentId = randomAlphaOfLength(10); + var model1 = mock(ElasticsearchInternalModel.class); + when(model1.mlNodeDeploymentId()).thenReturn(deploymentId); + when(model1.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); + var model2 = mock(ElasticsearchInternalModel.class); + when(model2.mlNodeDeploymentId()).thenReturn(deploymentId); + when(model2.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); + + var modelsByDeploymentId = new HashMap>(); + modelsByDeploymentId.put(deploymentId, List.of(model1, model2)); + + testUpdateModelsWithDynamicFields(modelsByDeploymentId); + } + + @SuppressWarnings("unchecked") + private void testUpdateModelsWithDynamicFields(Map> modelsByDeploymentId) throws IOException { + var modelsToUpdate = new ArrayList(); + modelsByDeploymentId.values().forEach(modelsToUpdate::addAll); + + var updatedNumberOfAllocations = new HashMap(); + modelsByDeploymentId.keySet().forEach(deploymentId -> updatedNumberOfAllocations.put(deploymentId, randomIntBetween(1, 10))); + + ActionListener> resultsListener = ActionListener.wrap(updatedModels -> { + assertEquals(updatedModels.size(), modelsToUpdate.size()); + modelsByDeploymentId.forEach((deploymentId, models) -> { + var expectedNumberOfAllocations = updatedNumberOfAllocations.get(deploymentId); + models.forEach(model -> { + verify((ElasticsearchInternalModel) model).updateNumAllocations(expectedNumberOfAllocations); + verify((ElasticsearchInternalModel) model).mlNodeDeploymentId(); + verifyNoMoreInteractions(model); + }); + }); + }, e -> fail("Unexpected exception: " + e)); + + var client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocation -> { + var listener = (ActionListener) invocation.getArguments()[2]; + var mockAssignmentStats = new ArrayList(); + modelsByDeploymentId.keySet().forEach(deploymentId -> { + var mockAssignmentStatsForDeploymentId = mock(AssignmentStats.class); + when(mockAssignmentStatsForDeploymentId.getDeploymentId()).thenReturn(deploymentId); + when(mockAssignmentStatsForDeploymentId.getNumberOfAllocations()).thenReturn(updatedNumberOfAllocations.get(deploymentId)); + mockAssignmentStats.add(mockAssignmentStatsForDeploymentId); + }); + listener.onResponse( + new GetDeploymentStatsAction.Response( + Collections.emptyList(), + Collections.emptyList(), + mockAssignmentStats, + mockAssignmentStats.size() + ) + ); + return null; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + + try (var service = createService(client)) { + service.updateModelsWithDynamicFields(modelsToUpdate, resultsListener); + } + } + + public void testUpdateWithoutMlEnabled() throws IOException, InterruptedException { + var cs = mock(ClusterService.class); + var cSettings = new ClusterSettings(Settings.EMPTY, Set.of(MachineLearningField.MAX_LAZY_ML_NODES)); + when(cs.getClusterSettings()).thenReturn(cSettings); + var context = new InferenceServiceExtension.InferenceServiceFactoryContext( + mock(), + threadPool, + cs, + Settings.builder().put("xpack.ml.enabled", false).build() + ); + try (var service = new ElasticsearchInternalService(context)) { + var models = List.of(mock(Model.class)); + var latch = new CountDownLatch(1); + service.updateModelsWithDynamicFields(models, ActionTestUtils.assertNoFailureListener(r -> { + latch.countDown(); + assertThat(r, Matchers.sameInstance(models)); + })); + assertTrue(latch.await(30, TimeUnit.SECONDS)); + } + } + + public void testUpdateWithMlEnabled() throws IOException, InterruptedException { + var deploymentId = "deploymentId"; + var model = mock(ElasticsearchInternalModel.class); + when(model.mlNodeDeploymentId()).thenReturn(deploymentId); + + AssignmentStats stats = mock(); + when(stats.getDeploymentId()).thenReturn(deploymentId); + when(stats.getNumberOfAllocations()).thenReturn(3); + + var client = mock(Client.class); + doAnswer(ans -> { + QueryPage queryPage = new QueryPage<>(List.of(stats), 1, RESULTS_FIELD); + + GetDeploymentStatsAction.Response response = mock(); + when(response.getStats()).thenReturn(queryPage); + + ActionListener listener = ans.getArgument(2); + listener.onResponse(response); + return null; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + when(client.threadPool()).thenReturn(threadPool); + + var cs = mock(ClusterService.class); + var cSettings = new ClusterSettings(Settings.EMPTY, Set.of(MachineLearningField.MAX_LAZY_ML_NODES)); + when(cs.getClusterSettings()).thenReturn(cSettings); + var context = new InferenceServiceExtension.InferenceServiceFactoryContext( + client, + threadPool, + cs, + Settings.builder().put("xpack.ml.enabled", true).build() + ); + try (var service = new ElasticsearchInternalService(context)) { + List models = List.of(model); + var latch = new CountDownLatch(1); + service.updateModelsWithDynamicFields(models, ActionTestUtils.assertNoFailureListener(r -> latch.countDown())); + assertTrue(latch.await(30, TimeUnit.SECONDS)); + verify(model).updateNumAllocations(3); + } + } + private ElasticsearchInternalService createService(Client client) { var cs = mock(ClusterService.class); var cSettings = new ClusterSettings(Settings.EMPTY, Set.of(MachineLearningField.MAX_LAZY_ML_NODES)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java index 96cd42efa42f5..5b21717ac03e4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java @@ -16,7 +16,7 @@ public void testUpdateNumAllocation() { "foo", TaskType.SPARSE_EMBEDDING, ElasticsearchInternalService.NAME, - new ElserInternalServiceSettings(null, 1, "elser", null), + new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(null, 1, "elser", null, null)), new ElserMlNodeTaskSettings(), null ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java index f4e97b2c2e5e0..dd4513db0d50a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java @@ -24,12 +24,12 @@ public static ElserInternalServiceSettings createRandom() { public void testBwcWrite() throws IOException { { - var settings = new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, ".elser_model_1", null)); + var settings = new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, ".elser_model_1", null, null)); var copy = copyInstance(settings, TransportVersions.V_8_12_0); assertEquals(settings, copy); } { - var settings = new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, ".elser_model_1", null)); + var settings = new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, ".elser_model_1", null, null)); var copy = copyInstance(settings, TransportVersions.V_8_11_X); assertEquals(settings, copy); } @@ -53,6 +53,7 @@ protected ElserInternalServiceSettings mutateInstance(ElserInternalServiceSettin instance.getNumAllocations() == null ? 1 : instance.getNumAllocations() + 1, instance.getNumThreads(), instance.modelId(), + null, null ) ); @@ -61,6 +62,7 @@ protected ElserInternalServiceSettings mutateInstance(ElserInternalServiceSettin instance.getNumAllocations(), instance.getNumThreads() + 1, instance.modelId(), + null, null ) ); @@ -72,6 +74,7 @@ yield new ElserInternalServiceSettings( instance.getNumAllocations(), instance.getNumThreads(), versions.iterator().next(), + null, null ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java index ff99101fc4ee5..99b7b3868b7f4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModel; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank.IbmWatsonxRerankModel; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; @@ -154,6 +155,42 @@ public void testParseRequestConfig_CreatesAIbmWatsonxEmbeddingsModel() throws IO } } + public void testParseRequestConfig_CreatesAIbmWatsonxRerankModel() throws IOException { + try (var service = createIbmWatsonxService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(IbmWatsonxRerankModel.class)); + + var rerankModel = (IbmWatsonxRerankModel) model; + assertThat(rerankModel.getServiceSettings().modelId(), is(modelId)); + assertThat(rerankModel.getServiceSettings().projectId(), is(projectId)); + assertThat(rerankModel.getServiceSettings().apiVersion(), is(apiVersion)); + assertThat(rerankModel.getSecretSettings().apiKey().toString(), is(apiKey)); + }, e -> fail("Model parsing should have succeeded, but failed: " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.RERANK, + getRequestConfigMap( + new HashMap<>( + Map.of( + ServiceFields.MODEL_ID, + modelId, + IbmWatsonxServiceFields.PROJECT_ID, + projectId, + ServiceFields.URL, + url, + IbmWatsonxServiceFields.API_VERSION, + apiVersion + ) + ), + new HashMap<>(Map.of()), + getSecretSettingsMap(apiKey) + ), + modelListener + ); + } + } + public void testParseRequestConfig_CreatesAIbmWatsonxEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { try (var service = createIbmWatsonxService()) { ActionListener modelListener = ActionListener.wrap(model -> { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankModelTests.java new file mode 100644 index 0000000000000..0138952c11e07 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/rerank/IbmWatsonxRerankModelTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.ibmwatsonx.rerank; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; + +import java.net.URI; + +public class IbmWatsonxRerankModelTests extends ESTestCase { + public static IbmWatsonxRerankModel createModel(String model, String projectId, URI uri, String apiVersion, String apiKey) { + return new IbmWatsonxRerankModel( + "id", + TaskType.RERANK, + "service", + new IbmWatsonxRerankServiceSettings(uri, apiVersion, model, projectId, null), + new IbmWatsonxRerankTaskSettings(2, true, 100), + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java index 5fa14da4ba733..2aeb0447f9c78 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java @@ -1831,33 +1831,53 @@ public void testDefaultSimilarity() { @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createJinaAIService()) { - String content = XContentHelper.stripWhitespace(""" - { - "service": "jinaai", - "name": "Jina AI", - "task_types": ["text_embedding", "rerank"], - "configurations": { - "api_key": { - "description": "API Key for the provider you're connecting to.", - "label": "API Key", - "required": true, - "sensitive": true, - "updatable": true, - "type": "str", - "supported_task_types": ["text_embedding", "rerank"] - }, - "rate_limit.requests_per_minute": { - "description": "Minimize the number of rate limit errors.", - "label": "Rate Limit", - "required": false, - "sensitive": false, - "updatable": false, - "type": "int", - "supported_task_types": ["text_embedding", "rerank"] + String content = XContentHelper.stripWhitespace( + """ + { + "service": "jinaai", + "name": "Jina AI", + "task_types": ["text_embedding", "rerank"], + "configurations": { + "api_key": { + "description": "API Key for the provider you're connecting to.", + "label": "API Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str", + "supported_task_types": ["text_embedding", "rerank"] + }, + "dimensions": { + "description": "The number of dimensions the resulting embeddings should have. For more information refer to https://api.jina.ai/redoc#tag/embeddings/operation/create_embedding_v1_embeddings_post.", + "label": "Dimensions", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding"] + }, + "model_id": { + "description": "The name of the model to use for the inference task.", + "label": "Model ID", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "rerank"] + }, + "rate_limit.requests_per_minute": { + "description": "Minimize the number of rate limit errors.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding", "rerank"] + } } } - } - """); + """ + ); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index 770b97186c588..77d4ec1f1bc87 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -28,14 +29,17 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.http.MockResponse; import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -59,10 +63,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.ExceptionsHelper.unwrapCause; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; import static org.elasticsearch.xpack.inference.Utils.getRequestConfigMap; @@ -84,6 +91,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.isA; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -931,7 +939,7 @@ public void testInfer_ThrowsErrorWhenTaskTypeIsNotValid_ChatCompletion() throws "Inference entity [model_id] does not support task type [chat_completion] " + "for inference, the task type must be one of [text_embedding, completion]. " + "The task type for the inference entity is chat_completion, " - + "please use the _inference/chat_completion/model_id/_unified URL." + + "please use the _inference/chat_completion/model_id/_stream URL." ) ); @@ -1061,6 +1069,125 @@ public void testUnifiedCompletionInfer() throws Exception { } } + public void testUnifiedCompletionError() throws Exception { + String responseJson = """ + { + "error": { + "message": "The model `gpt-4awero` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + } + }"""; + webServer.enqueue(new MockResponse().setResponseCode(404).setBody(responseJson)); + + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var model = OpenAiChatCompletionModelTests.createChatCompletionModel(getUrl(webServer), "org", "secret", "model", "user"); + var latch = new CountDownLatch(1); + service.unifiedCompletionInfer( + model, + UnifiedCompletionRequest.of( + List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "user", null, null)) + ), + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.runAfter(ActionTestUtils.assertNoSuccessListener(e -> { + try (var builder = XContentFactory.jsonBuilder()) { + var t = unwrapCause(e); + assertThat(t, isA(UnifiedChatCompletionException.class)); + ((UnifiedChatCompletionException) t).toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + var json = XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); + + assertThat(json, is(""" + {\ + "error":{\ + "code":"model_not_found",\ + "message":"Received an unsuccessful status code for request from inference entity id [id] status \ + [404]. Error message: [The model `gpt-4awero` does not exist or you do not have access to it.]",\ + "type":"invalid_request_error"\ + }}""")); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }), latch::countDown) + ); + assertTrue(latch.await(30, TimeUnit.SECONDS)); + } + } + + public void testMidStreamUnifiedCompletionError() throws Exception { + String responseJson = """ + event: error + data: { "error": { "message": "Timed out waiting for more data", "type": "timeout" } } + + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + testStreamError(""" + {\ + "error":{\ + "message":"Received an error response for request from inference entity id [id]. Error message: \ + [Timed out waiting for more data]",\ + "type":"timeout"\ + }}"""); + } + + private void testStreamError(String expectedResponse) throws Exception { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var model = OpenAiChatCompletionModelTests.createChatCompletionModel(getUrl(webServer), "org", "secret", "model", "user"); + PlainActionFuture listener = new PlainActionFuture<>(); + service.unifiedCompletionInfer( + model, + UnifiedCompletionRequest.of( + List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "user", null, null)) + ), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoEvents().hasErrorMatching(e -> { + e = unwrapCause(e); + assertThat(e, isA(UnifiedChatCompletionException.class)); + try (var builder = XContentFactory.jsonBuilder()) { + ((UnifiedChatCompletionException) e).toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + var json = XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); + + assertThat(json, is(expectedResponse)); + } + }); + } + } + + public void testUnifiedCompletionMalformedError() throws Exception { + String responseJson = """ + data: { invalid json } + + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + testStreamError(""" + {\ + "error":{\ + "code":"bad_request",\ + "message":"[1:3] Unexpected character ('i' (code 105)): was expecting double-quote to start field name\\n\ + at [Source: (String)\\"{ invalid json }\\"; line: 1, column: 3]",\ + "type":"x_content_parse_exception"\ + }}"""); + } + public void testInfer_StreamRequest() throws Exception { String responseJson = """ data: {\ @@ -1084,13 +1211,11 @@ public void testInfer_StreamRequest() throws Exception { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var result = streamCompletion(); - - InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + streamCompletion().hasNoErrors().hasEvent(""" {"completion":[{"delta":"hello, world"}]}"""); } - private InferenceServiceResults streamCompletion() throws IOException { + private InferenceEventsAssertion streamCompletion() throws Exception { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { var model = OpenAiChatCompletionModelTests.createCompletionModel(getUrl(webServer), "org", "secret", "model", "user"); @@ -1106,7 +1231,7 @@ private InferenceServiceResults streamCompletion() throws IOException { listener ); - return listener.actionGet(TIMEOUT); + return InferenceEventsAssertion.assertThat(listener.actionGet(TIMEOUT)).hasFinishedStream(); } } @@ -1122,13 +1247,48 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { }"""; webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); - var result = streamCompletion(); + var e = assertThrows(ElasticsearchStatusException.class, this::streamCompletion); + assertThat(e.status(), equalTo(RestStatus.UNAUTHORIZED)); + assertThat( + e.getMessage(), + equalTo( + "Received an authentication error status code for request from inference entity id [id] status [401]. " + + "Error message: [You didn't provide an API key...]" + ) + ); + } - InferenceEventsAssertion.assertThat(result) - .hasFinishedStream() - .hasNoEvents() - .hasErrorWithStatusCode(401) - .hasErrorContaining("You didn't provide an API key..."); + public void testInfer_StreamRequestRetry() throws Exception { + webServer.enqueue(new MockResponse().setResponseCode(503).setBody(""" + { + "error": { + "message": "server busy", + "type": "server_busy" + } + }""")); + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(""" + data: {\ + "id":"12345",\ + "object":"chat.completion.chunk",\ + "created":123456789,\ + "model":"gpt-4o-mini",\ + "system_fingerprint": "123456789",\ + "choices":[\ + {\ + "index":0,\ + "delta":{\ + "content":"hello, world"\ + },\ + "logprobs":null,\ + "finish_reason":null\ + }\ + ]\ + } + + """)); + + streamCompletion().hasNoErrors().hasEvent(""" + {"completion":[{"delta":"hello, world"}]}"""); } public void testSupportsStreaming() throws IOException { @@ -1751,6 +1911,15 @@ public void testGetConfiguration() throws Exception { "type": "str", "supported_task_types": ["text_embedding", "completion", "chat_completion"] }, + "dimensions": { + "description": "The number of dimensions the resulting embeddings should have. For more information refer to https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-dimensions.", + "label": "Dimensions", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding"] + }, "organization_id": { "description": "The unique identifier of your organization.", "label": "Organization ID", @@ -1777,16 +1946,6 @@ public void testGetConfiguration() throws Exception { "updatable": false, "type": "str", "supported_task_types": ["text_embedding", "completion", "chat_completion"] - }, - "url": { - "default_value": "https://api.openai.com/v1/chat/completions", - "description": "The OpenAI API endpoint URL. For more information on the URL, refer to the https://platform.openai.com/docs/api-reference.", - "label": "URL", - "required": true, - "sensitive": false, - "updatable": false, - "type": "str", - "supported_task_types": ["text_embedding", "completion", "chat_completion"] } } } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml index 07341273151bc..5f87942b2c710 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml @@ -650,3 +650,28 @@ setup: - match: { hits.total.value: 1 } - match: { hits.total.relation: eq } - match: { hits.hits.0._source.dense_field.text: "updated text" } + +--- +"Skip fetching _inference_fields": + - requires: + cluster_features: semantic_text.skip_inference_fields + reason: Skip _inference_fields when search is performed on legacy semantic_text format. + + - do: + index: + index: test-index + id: doc_1 + body: + sparse_field: "test value" + refresh: true + + - do: + search: + index: test-index + body: + fields: [ _inference_fields ] + query: + match_all: { } + + - match: { hits.total.value: 1 } + - not_exists: hits.hits.0._source._inference_fields diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml index dec4e127e501c..64ecb0f2d882c 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/47_semantic_text_knn.yml @@ -43,6 +43,8 @@ setup: body: mappings: properties: + keyword_field: + type: keyword inference_field: type: semantic_text inference_id: dense-inference-id @@ -53,6 +55,8 @@ setup: body: mappings: properties: + keyword_field: + type: keyword inference_field: type: semantic_text inference_id: dense-inference-id-2 @@ -63,6 +67,8 @@ setup: body: mappings: properties: + keyword_field: + type: keyword inference_field: type: dense_vector dims: 10 @@ -74,6 +80,8 @@ setup: body: mappings: properties: + keyword_field: + type: keyword inference_field: type: dense_vector dims: 3 @@ -84,6 +92,7 @@ setup: index: test-semantic-text-index id: doc_1 body: + keyword_field: "foo" inference_field: [ "inference test", "another inference test" ] refresh: true @@ -92,6 +101,7 @@ setup: index: test-semantic-text-index-2 id: doc_2 body: + keyword_field: "bar" inference_field: [ "inference test", "another inference test" ] refresh: true @@ -100,6 +110,7 @@ setup: index: test-dense-vector-index id: doc_3 body: + keyword_field: "baz" inference_field: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] refresh: true @@ -108,6 +119,7 @@ setup: index: test-incompatible-dense-vector-index id: doc_4 body: + keyword_field: "qux" inference_field: [ 1, 2, 3 ] refresh: true @@ -311,6 +323,34 @@ setup: - match: { hits.total.value: 2 } +--- +"knn query respects filters": + - requires: + cluster_features: "search.semantic_knn_filter_fix" + reason: filters fixed in 8.18.0 + + - do: + search: + index: + - test-semantic-text-index + - test-semantic-text-index-2 + body: + query: + knn: + field: inference_field + k: 10 + num_candidates: 100 + query_vector_builder: + text_embedding: + model_text: test + filter: + term: + keyword_field: "foo" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + + --- "knn query against multiple semantic_text fields with multiple inference IDs specified in semantic_text fields with smaller k returns k for each index": diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java index 4a9d13bc642d7..56703ebe9c308 100644 --- a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java +++ b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java @@ -257,7 +257,7 @@ public void testLogsdbRouteOnSortFields() throws IOException { var settings = (Map) ((Map) getIndexSettings(index).get(index)).get("settings"); assertEquals("logsdb", settings.get("index.mode")); assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode")); - assertEquals("true", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey())); - assertEquals(List.of("host.name", "message"), settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey())); + assertEquals("false", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey())); + assertNull(settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey())); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java index 320c2775cefc3..8d7b47f900592 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java @@ -520,6 +520,7 @@ private void assertDeprecationWarningForTemplate(String templateName) throws IOE Map issuesByTemplate = (Map) response.get("templates"); assertThat(issuesByTemplate.containsKey(templateName), equalTo(true)); var templateIssues = (List) issuesByTemplate.get(templateName); - assertThat(((Map) templateIssues.get(0)).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); + assertThat(((Map) templateIssues.get(0)).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING_TITLE)); + assertThat(((Map) templateIssues.get(0)).get("details"), equalTo(SourceFieldMapper.DEPRECATION_WARNING)); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java index 1f14d13100330..7c2e6a2f374f1 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class LogsdbRestIT extends ESRestTestCase { @@ -73,9 +74,15 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException { List> features = (List>) response.get("features"); logger.info("response's features: {}", features); assertThat(features, Matchers.not(Matchers.empty())); - Map feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get(); - assertThat(feature.get("name"), equalTo("synthetic-source")); - assertThat(feature.get("license_level"), equalTo("enterprise")); + boolean found = false; + for (var feature : features) { + if (feature.get("family") != null) { + assertThat(feature.get("name"), anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields"))); + assertThat(feature.get("license_level"), equalTo("enterprise")); + found = true; + } + } + assertTrue(found); var settings = (Map) ((Map) getIndexSettings("test-index").get("test-index")).get("settings"); assertNull(settings.get("index.mapping.source.mode")); // Default, no downgrading. diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java new file mode 100644 index 0000000000000..7448304c29992 --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbSnapshotRestoreIT.java @@ -0,0 +1,369 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.net.InetAddress; +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasSize; + +public class LogsdbSnapshotRestoreIT extends ESRestTestCase { + + private static TemporaryFolder repoDirectory = new TemporaryFolder(); + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("path.repo", () -> getRepoPath()) + .setting("cluster.logsdb.enabled", "true") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + + static final String LOGS_TEMPLATE = """ + { + "index_patterns": [ "logs-*-*" ], + "data_stream": {}, + "priority": 1000, + "template": { + "settings": { + "index": { + "mapping": { + "source":{ + "mode": "{{source_mode}}" + } + } + } + }, + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "pid": { + "type": "integer" + }, + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "ip_address": { + "type": "ip" + }, + "my_object_array": { + "type": "{{array_type}}" + } + } + } + } + }"""; + + static final String DOC_TEMPLATE = """ + { + "@timestamp": "%s", + "host": { "name": "%s"}, + "pid": %d, + "method": "%s", + "message": "%s", + "ip_address": "%s", + "memory_usage_bytes": "%d", + "my_object_array": [ + { + "field_1": "a", + "field_2": "b" + }, + { + "field_1": "c", + "field_2": "d" + } + ] + } + """; + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testSnapshotRestore() throws Exception { + snapshotAndRestore("synthetic", "object", false); + } + + public void testSnapshotRestoreWithSourceOnlyRepository() throws Exception { + snapshotAndFail("object"); + } + + public void testSnapshotRestoreNested() throws Exception { + snapshotAndRestore("synthetic", "nested", false); + } + + public void testSnapshotRestoreNestedWithSourceOnlyRepository() throws Exception { + snapshotAndFail("nested"); + } + + public void testSnapshotRestoreStoredSource() throws Exception { + snapshotAndRestore("stored", "object", false); + } + + public void testSnapshotRestoreStoredSourceWithSourceOnlyRepository() throws Exception { + snapshotAndRestore("stored", "object", true); + } + + public void testSnapshotRestoreStoredSourceNested() throws Exception { + snapshotAndRestore("stored", "nested", false); + } + + public void testSnapshotRestoreStoredSourceNestedWithSourceOnlyRepository() throws Exception { + snapshotAndRestore("stored", "nested", true); + } + + @After + public void cleanup() throws Exception { + deleteSnapshot("my-repository", "my-snapshot", true); + deleteRepository("my-repository"); + deleteDataStream("logs-my-test"); + } + + static void snapshotAndRestore(String sourceMode, String arrayType, boolean sourceOnly) throws IOException { + String dataStreamName = "logs-my-test"; + String repositoryName = "my-repository"; + if (sourceOnly) { + var repositorySettings = Settings.builder().put("delegate_type", "fs").put("location", getRepoPath()).build(); + registerRepository(repositoryName, "source", true, repositorySettings); + } else { + var repositorySettings = Settings.builder().put("location", getRepoPath()).build(); + registerRepository(repositoryName, FsRepository.TYPE, true, repositorySettings); + } + + putTemplate("my-template", LOGS_TEMPLATE.replace("{{source_mode}}", sourceMode).replace("{{array_type}}", arrayType)); + String[] docs = new String[100]; + for (int i = 0; i < 100; i++) { + docs[i] = document( + Instant.now(), + String.format(Locale.ROOT, "host-%03d", i), + randomNonNegativeInt(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(32), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ); + indexDocument(dataStreamName, docs[i]); + } + refresh(dataStreamName); + assertDocCount(client(), dataStreamName, 100); + assertSource(dataStreamName, docs); + assertDataStream(dataStreamName, sourceMode); + + String snapshotName = "my-snapshot"; + var snapshotResponse = performSnapshot(repositoryName, dataStreamName, snapshotName, true); + assertOK(snapshotResponse); + var snapshotResponseBody = entityAsMap(snapshotResponse); + Map snapshotItem = (Map) snapshotResponseBody.get("snapshot"); + List failures = (List) snapshotItem.get("failures"); + assertThat(failures, empty()); + deleteDataStream(dataStreamName); + assertDocCount(dataStreamName, 0); + + restoreSnapshot(repositoryName, snapshotName, true); + assertDataStream(dataStreamName, sourceMode); + assertDocCount(dataStreamName, 100); + assertSource(dataStreamName, docs); + } + + static void snapshotAndFail(String arrayType) throws IOException { + String dataStreamName = "logs-my-test"; + String repositoryName = "my-repository"; + var repositorySettings = Settings.builder().put("delegate_type", "fs").put("location", getRepoPath()).build(); + registerRepository(repositoryName, "source", true, repositorySettings); + + putTemplate("my-template", LOGS_TEMPLATE.replace("{{source_mode}}", "synthetic").replace("{{array_type}}", arrayType)); + for (int i = 0; i < 100; i++) { + indexDocument( + dataStreamName, + document( + Instant.now(), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(32), + randomIp(randomBoolean()), + randomIntBetween(1_000_000, 2_000_000) + ) + ); + } + refresh(dataStreamName); + assertDocCount(client(), dataStreamName, 100); + assertDataStream(dataStreamName, "synthetic"); + + String snapshotName = "my-snapshot"; + var snapshotResponse = performSnapshot(repositoryName, dataStreamName, snapshotName, true); + assertOK(snapshotResponse); + var snapshotResponseBody = entityAsMap(snapshotResponse); + Map snapshotItem = (Map) snapshotResponseBody.get("snapshot"); + List failures = (List) snapshotItem.get("failures"); + assertThat(failures, hasSize(1)); + Map failure = (Map) failures.get(0); + assertThat( + (String) failure.get("reason"), + containsString( + "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source" + ) + ); + } + + static void deleteDataStream(String dataStreamName) throws IOException { + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/" + dataStreamName))); + } + + static void putTemplate(String templateName, String template) throws IOException { + final Request request = new Request("PUT", "/_index_template/" + templateName); + request.setJsonEntity(template); + assertOK(client().performRequest(request)); + } + + static void indexDocument(String indexOrtDataStream, String doc) throws IOException { + final Request request = new Request("POST", "/" + indexOrtDataStream + "/_doc?refresh=true"); + request.setJsonEntity(doc); + final Response response = client().performRequest(request); + assertOK(response); + assertThat(entityAsMap(response).get("result"), equalTo("created")); + } + + static String document( + final Instant timestamp, + final String hostname, + long pid, + final String method, + final String message, + final InetAddress ipAddress, + long memoryUsageBytes + ) { + return String.format( + Locale.ROOT, + DOC_TEMPLATE, + DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(timestamp), + hostname, + pid, + method, + message, + InetAddresses.toAddrString(ipAddress), + memoryUsageBytes + ); + } + + static Response performSnapshot(String repository, String dataStreamName, String snapshot, boolean waitForCompletion) + throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); + request.setJsonEntity(""" + { + "indices": "{{dataStreamName}}" + } + """.replace("{{dataStreamName}}", dataStreamName)); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + + return client().performRequest(request); + } + + static void assertDataStream(String dataStreamName, final String sourceMode) throws IOException { + String indexName = getWriteBackingIndex(dataStreamName, 0); + var flatSettings = (Map) ((Map) getIndexSettings(indexName).get(indexName)).get("settings"); + assertThat(flatSettings, hasEntry("index.mode", "logsdb")); + assertThat(flatSettings, hasEntry("index.mapping.source.mode", sourceMode)); + } + + static String getWriteBackingIndex(String dataStreamName, int backingIndex) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client().performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List backingIndices = (List) dataStream.get("indices"); + return (String) ((Map) backingIndices.get(backingIndex)).get("index_name"); + } + + static void assertDocCount(String indexName, long docCount) throws IOException { + Request countReq = new Request("GET", "/" + indexName + "/_count"); + countReq.addParameter("ignore_unavailable", "true"); + ObjectPath resp = ObjectPath.createFromResponse(client().performRequest(countReq)); + assertEquals( + "expected " + docCount + " documents but it was a different number", + docCount, + Long.parseLong(resp.evaluate("count").toString()) + ); + } + + static void assertSource(String indexName, String[] docs) throws IOException { + Request searchReq = new Request("GET", "/" + indexName + "/_search"); + searchReq.addParameter("size", String.valueOf(docs.length)); + var response = client().performRequest(searchReq); + assertOK(response); + var responseBody = entityAsMap(response); + List hits = (List) ((Map) responseBody.get("hits")).get("hits"); + assertThat(hits, hasSize(docs.length)); + for (Object hit : hits) { + Map actualSource = (Map) ((Map) hit).get("_source"); + String actualHost = (String) ((Map) actualSource.get("host")).get("name"); + Map expectedSource = null; + for (String doc : docs) { + expectedSource = XContentHelper.convertToMap(XContentType.JSON.xContent(), doc, false); + String expectedHost = (String) ((Map) expectedSource.get("host")).get("name"); + if (expectedHost.equals(actualHost)) { + break; + } + } + + assertMap(actualSource, matchesMap(expectedSource)); + } + } + + @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") + private static String getRepoPath() { + return repoDirectory.getRoot().getPath(); + } + +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index 81dbf1359c2db..cc4c32bcbc27b 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -31,12 +31,12 @@ import static org.elasticsearch.xpack.logsdb.LogsPatternUsageService.LOGSDB_PRIOR_LOGS_USAGE; import static org.elasticsearch.xpack.logsdb.LogsPatternUsageService.USAGE_CHECK_MAX_PERIOD; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING; +import static org.elasticsearch.xpack.logsdb.LogsdbLicenseService.FALLBACK_SETTING; public class LogsDBPlugin extends Plugin implements ActionPlugin { private final Settings settings; - private final SyntheticSourceLicenseService licenseService; + private final LogsdbLicenseService licenseService; public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( "cluster.logsdb.enabled", false, @@ -48,7 +48,7 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin { public LogsDBPlugin(Settings settings) { this.settings = settings; - this.licenseService = new SyntheticSourceLicenseService(settings); + this.licenseService = new LogsdbLicenseService(settings); this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(licenseService, settings); } @@ -87,6 +87,8 @@ public Collection getAdditionalIndexSettingProviders(Index IndexVersion.current(), parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion() ), + () -> parameters.clusterService().state().nodes().getMinNodeVersion(), + DiscoveryNode.isStateless(settings) == false, DiscoveryNode.isStateless(settings) == false ); return List.of(logsdbIndexModeSettingsProvider); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java index 4a23ac89e3805..9402c8edc4bc8 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java @@ -69,7 +69,7 @@ protected void masterOperation( } } final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings()); - final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null; + final boolean hasCustomCutoffDate = System.getProperty(LogsdbLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null; if (featureService.clusterHasFeature(state, XPackFeatures.LOGSDB_TELMETRY_STATS)) { final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new); final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index ac19c96f31b5c..0257f2c3da8be 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.UUIDs; @@ -49,15 +50,17 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { static final String LOGS_PATTERN = "logs-*-*"; private static final Set MAPPING_INCLUDES = Set.of("_doc._source.*", "_doc.properties.host**", "_doc.subobjects"); - private final SyntheticSourceLicenseService syntheticSourceLicenseService; + private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); private final SetOnce> createdIndexVersion = new SetOnce<>(); + private final SetOnce> minNodeVersion = new SetOnce<>(); private final SetOnce supportFallbackToStoredSource = new SetOnce<>(); + private final SetOnce supportFallbackLogsdbRouting = new SetOnce<>(); private volatile boolean isLogsdbEnabled; - LogsdbIndexModeSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService, final Settings settings) { - this.syntheticSourceLicenseService = syntheticSourceLicenseService; + LogsdbIndexModeSettingsProvider(LogsdbLicenseService licenseService, final Settings settings) { + this.licenseService = licenseService; this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings); } @@ -68,11 +71,15 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { void init( CheckedFunction factory, Supplier indexVersion, - boolean supportFallbackToStoredSource + Supplier minNodeVersion, + boolean supportFallbackToStoredSource, + boolean supportFallbackLogsdbRouting ) { this.mapperServiceFactory.set(factory); this.createdIndexVersion.set(indexVersion); + this.minNodeVersion.set(minNodeVersion); this.supportFallbackToStoredSource.set(supportFallbackToStoredSource); + this.supportFallbackLogsdbRouting.set(supportFallbackLogsdbRouting); } @Override @@ -93,6 +100,7 @@ public Settings getAdditionalIndexSettings( ) { Settings.Builder settingsBuilder = null; boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB; + boolean isTemplateValidation = "validate-index-name".equals(indexName); // Inject logsdb index mode, based on the logs pattern. if (isLogsdbEnabled @@ -107,79 +115,79 @@ && matchesLogsPattern(dataStreamName)) { MappingHints mappingHints = getMappingHints(indexName, templateIndexMode, settings, combinedTemplateMappings); // Inject stored source mode if synthetic source if not available per licence. - if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) { + if (mappingHints.hasSyntheticSourceUsage + && supportFallbackToStoredSource.get() + && minNodeVersion.get().get().onOrAfter(Version.V_8_17_0)) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) - boolean isTemplateValidation = "validate-index-name".equals(indexName); boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed( templateIndexMode, indexName, dataStreamName ); - if (syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) { + if (licenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) { LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName); - if (settingsBuilder == null) { - settingsBuilder = Settings.builder(); - } - settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); + settingsBuilder = getBuilder(settingsBuilder).put( + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + SourceFieldMapper.Mode.STORED.toString() + ); } } - if (isLogsDB) { + if (isLogsDB && minNodeVersion.get().get().onOrAfter(Version.V_8_18_0)) { // Inject sorting on [host.name], in addition to [@timestamp]. if (mappingHints.sortOnHostName) { - if (settingsBuilder == null) { - settingsBuilder = Settings.builder(); - } if (mappingHints.addHostNameField) { // Inject keyword field [host.name] too. - settingsBuilder.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true); + settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true); } - settingsBuilder.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true); + settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true); } // Inject routing path matching sort fields. if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) { - List sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())); - sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH)); - if (sortFields.size() < 2) { - throw new IllegalStateException( - String.format( - Locale.ROOT, - "data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields " - + "(excluding timestamp), needs at least 2", - dataStreamName, - IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), - sortFields.size() - ) - ); - } - if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) { - List routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()); - if (routingPaths.equals(sortFields) == false) { + if (supportFallbackLogsdbRouting.get() == false || licenseService.allowLogsdbRoutingOnSortField(isTemplateValidation)) { + List sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())); + sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH)); + if (sortFields.size() < 2) { throw new IllegalStateException( String.format( Locale.ROOT, - "data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort " - + "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]", + "data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields " + + "(excluding timestamp), needs at least 2", dataStreamName, IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), - routingPaths, - sortFields + sortFields.size() ) ); } - } else { - if (settingsBuilder == null) { - settingsBuilder = Settings.builder(); + if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) { + List routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()); + if (routingPaths.equals(sortFields) == false) { + throw new IllegalStateException( + String.format( + Locale.ROOT, + "data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort " + + "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]", + dataStreamName, + IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), + routingPaths, + sortFields + ) + ); + } + } else { + settingsBuilder = getBuilder(settingsBuilder).putList(INDEX_ROUTING_PATH.getKey(), sortFields); } - settingsBuilder.putList(INDEX_ROUTING_PATH.getKey(), sortFields); + } else { + // Routing on sort fields is not allowed, reset the corresponding index setting. + LOGGER.debug("creation of index [{}] with logsdb mode and routing on sort fields without it being allowed", indexName); + settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false); } } } return settingsBuilder == null ? Settings.EMPTY : settingsBuilder.build(); - } record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) { @@ -194,6 +202,14 @@ private static IndexMode resolveIndexMode(final String mode) { return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null; } + // Returned value needs to be reassigned to the passed arg, to track the created builder. + private static Settings.Builder getBuilder(Settings.Builder builder) { + if (builder == null) { + return Settings.builder(); + } + return builder; + } + MappingHints getMappingHints( String indexName, IndexMode templateIndexMode, @@ -260,8 +276,8 @@ MappingHints getMappingHints( || mapperService.mappingLookup().getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.DISABLED)); boolean sortOnHostName = IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings) || addHostNameField - || ((hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues()) - || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues())); + || (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues()) + || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()); return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField); } } catch (AssertionError | Exception e) { diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java similarity index 86% rename from x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java rename to x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java index f7f228859fb6d..d3487e205b33e 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java @@ -23,12 +23,12 @@ /** * Determines based on license and fallback setting whether synthetic source usages should fallback to stored source. */ -final class SyntheticSourceLicenseService { +final class LogsdbLicenseService { static final String MAPPINGS_FEATURE_FAMILY = "mappings"; // You can only override this property if you received explicit approval from Elastic. static final String CUTOFF_DATE_SYS_PROP_NAME = "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override"; - private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class); + private static final Logger LOGGER = LogManager.getLogger(LogsdbLicenseService.class); static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2025, 2, 4, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); /** @@ -53,16 +53,22 @@ final class SyntheticSourceLicenseService { License.OperationMode.GOLD ); + static final LicensedFeature.Momentary LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE = LicensedFeature.momentary( + MAPPINGS_FEATURE_FAMILY, + "logsdb-routing-on-sort-fields", + License.OperationMode.ENTERPRISE + ); + private final long cutoffDate; private LicenseService licenseService; private XPackLicenseState licenseState; private volatile boolean syntheticSourceFallback; - SyntheticSourceLicenseService(Settings settings) { + LogsdbLicenseService(Settings settings) { this(settings, System.getProperty(CUTOFF_DATE_SYS_PROP_NAME)); } - SyntheticSourceLicenseService(Settings settings, String cutoffDate) { + LogsdbLicenseService(Settings settings, String cutoffDate) { this.syntheticSourceFallback = FALLBACK_SETTING.get(settings); this.cutoffDate = getCutoffDate(cutoffDate); } @@ -97,6 +103,13 @@ && checkFeature(SYNTHETIC_SOURCE_FEATURE_LEGACY, licenseStateSnapshot, isTemplat return true; } + /** + * @return whether indexes in logsdb mode can use routing on sort fields. + */ + public boolean allowLogsdbRoutingOnSortField(boolean isTemplateValidation) { + return checkFeature(LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE, licenseState.copyCurrentLicenseState(), isTemplateValidation); + } + private static boolean checkFeature( LicensedFeature.Momentary licensedFeature, XPackLicenseState licenseStateSnapshot, diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java index b2a533f6b76c9..22462a2b22bd4 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java @@ -31,8 +31,8 @@ import java.util.List; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createGoldOrPlatinumLicense; +import static org.elasticsearch.xpack.logsdb.LogsdbLicenseServiceTests.createEnterpriseLicense; +import static org.elasticsearch.xpack.logsdb.LogsdbLicenseServiceTests.createGoldOrPlatinumLicense; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -57,15 +57,15 @@ public void setup() throws Exception { public void testSyntheticSourceUsageDisallowed() { createIndexWithSyntheticSourceAndAssertExpectedType("test", "STORED"); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue()); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue()); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue()); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue()); } public void testSyntheticSourceUsageWithLegacyLicense() { createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-stacktraces", "synthetic"); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue())); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue()); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue())); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue()); } public void testSyntheticSourceUsageWithLegacyLicensePastCutoff() throws Exception { @@ -75,8 +75,8 @@ public void testSyntheticSourceUsageWithLegacyLicensePastCutoff() throws Excepti ensureGreen(); createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-stacktraces", "STORED"); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue()); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue()); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue()); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE, nullValue()); } public void testSyntheticSourceUsageWithEnterpriseLicensePastCutoff() throws Exception { @@ -87,8 +87,8 @@ public void testSyntheticSourceUsageWithEnterpriseLicensePastCutoff() throws Exc createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces", "synthetic"); // also supports non-exceptional indices createIndexWithSyntheticSourceAndAssertExpectedType("test", "synthetic"); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue()); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue())); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, nullValue()); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue())); } public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws Exception { @@ -99,8 +99,8 @@ public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws E createIndexWithSyntheticSourceAndAssertExpectedType(".profiling-traces-v2", "synthetic"); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue())); - assertFeatureUsage(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue())); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY, not(nullValue())); + assertFeatureUsage(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE, not(nullValue())); } private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) { @@ -119,7 +119,7 @@ private List getFeatureUsageInfo() { private void assertFeatureUsage(LicensedFeature.Momentary syntheticSourceFeature, Matcher matcher) { GetFeatureUsageResponse.FeatureUsageInfo featureUsage = getFeatureUsageInfo().stream() - .filter(f -> f.getFamily().equals(SyntheticSourceLicenseService.MAPPINGS_FEATURE_FAMILY)) + .filter(f -> f.getFamily().equals(LogsdbLicenseService.MAPPINGS_FEATURE_FAMILY)) .filter(f -> f.getName().equals(syntheticSourceFeature.getName())) .findAny() .orElse(null); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 77319a881f1e5..44bc551241c4f 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.logsdb; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; import org.elasticsearch.cluster.metadata.DataStream; @@ -37,10 +38,12 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.settings.Settings.builder; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense; +import static org.elasticsearch.xpack.logsdb.LogsdbLicenseServiceTests.createEnterpriseLicense; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -65,50 +68,65 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { } """; - private SyntheticSourceLicenseService syntheticSourceLicenseService; + private LogsdbLicenseService logsdbLicenseService; private final AtomicInteger newMapperServiceCounter = new AtomicInteger(); @Before public void setup() throws Exception { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.isAllowed(any())).thenReturn(true); - var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); - licenseService.setLicenseState(licenseState); var mockLicenseService = mock(LicenseService.class); License license = createEnterpriseLicense(); when(mockLicenseService.getLicense()).thenReturn(license); - syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); - syntheticSourceLicenseService.setLicenseState(licenseState); - syntheticSourceLicenseService.setLicenseService(mockLicenseService); + logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY); + logsdbLicenseService.setLicenseState(licenseState); + logsdbLicenseService.setLicenseService(mockLicenseService); } private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { + return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT); + } + + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled, Version version) { newMapperServiceCounter.set(0); var provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", enabled).build() ); provider.init(im -> { newMapperServiceCounter.incrementAndGet(); return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); - }, IndexVersion::current, true); + }, IndexVersion::current, () -> version, true, true); + return provider; + } + + private LogsdbIndexModeSettingsProvider withoutMapperService(boolean enabled) { + var provider = new LogsdbIndexModeSettingsProvider( + logsdbLicenseService, + Settings.builder().put("cluster.logsdb.enabled", enabled).build() + ); + provider.init(im -> null, IndexVersion::current, () -> Version.CURRENT, true, true); return provider; } private Settings generateLogsdbSettings(Settings settings) throws IOException { - return generateLogsdbSettings(settings, null); + return generateLogsdbSettings(settings, null, Version.CURRENT); } private Settings generateLogsdbSettings(Settings settings, String mapping) throws IOException { + return generateLogsdbSettings(settings, mapping, Version.CURRENT); + } + + private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException { Metadata metadata = Metadata.EMPTY_METADATA; var provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); provider.init(im -> { newMapperServiceCounter.incrementAndGet(); return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); - }, IndexVersion::current, true); + }, IndexVersion::current, () -> version, true, true); var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0), DATA_STREAM_NAME, @@ -123,7 +141,7 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw public void testDisabled() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", false).build() ); @@ -142,7 +160,7 @@ public void testDisabled() throws IOException { public void testOnIndexCreation() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -161,7 +179,7 @@ public void testOnIndexCreation() throws IOException { public void testOnExplicitStandardIndex() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -180,7 +198,7 @@ public void testOnExplicitStandardIndex() throws IOException { public void testOnExplicitTimeSeriesIndex() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -199,7 +217,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { public void testNonLogsDataStream() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -217,11 +235,7 @@ public void testNonLogsDataStream() throws IOException { } public void testWithoutLogsComponentTemplate() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -236,11 +250,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { } public void testWithLogsComponentTemplate() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -255,11 +265,7 @@ public void testWithLogsComponentTemplate() throws IOException { } public void testWithMultipleComponentTemplates() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -274,11 +280,7 @@ public void testWithMultipleComponentTemplates() throws IOException { } public void testWithCustomComponentTemplatesOnly() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -293,11 +295,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { } public void testNonMatchingTemplateIndexPattern() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -313,7 +311,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { public void testCaseSensitivity() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, + logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -331,10 +329,7 @@ public void testCaseSensitivity() throws IOException { } public void testMultipleHyphensInDataStreamName() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, @@ -349,12 +344,8 @@ public void testMultipleHyphensInDataStreamName() throws IOException { assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); } - public void testBeforeAndAFterSettingUpdate() throws IOException { - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", false).build() - ); - + public void testBeforeAndAfterSettingUpdate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = withoutMapperService(false); final Settings beforeSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", @@ -583,7 +574,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { } } - public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOException { + public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOException { String dataStreamName = DATA_STREAM_NAME; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings settings = Settings.builder().put("index.soft_deletes.enabled", false).build(); @@ -627,7 +618,7 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep } } - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException { + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { String dataStreamName = DATA_STREAM_NAME; Metadata.Builder mb = Metadata.builder( DataStreamTestHelper.getClusterStateWithDataStreams( @@ -655,7 +646,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws assertThat(result.size(), equalTo(0)); assertThat(newMapperServiceCounter.get(), equalTo(1)); - syntheticSourceLicenseService.setSyntheticSourceFallback(true); + logsdbLicenseService.setSyntheticSourceFallback(true); result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, @@ -698,8 +689,35 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws assertThat(newMapperServiceCounter.get(), equalTo(4)); } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() { + logsdbLicenseService.setSyntheticSourceFallback(true); + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true, Version.V_8_16_0); + Metadata.Builder mb = Metadata.builder( + DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple(DATA_STREAM_NAME, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ).getMetadata() + ); + Metadata metadata = mb.build(); + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .build(); + var result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 2), + DATA_STREAM_NAME, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertTrue(result.isEmpty()); + } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException { - syntheticSourceLicenseService.setSyntheticSourceFallback(true); + logsdbLicenseService.setSyntheticSourceFallback(true); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true); final Settings settings = Settings.EMPTY; @@ -773,6 +791,15 @@ public void testRoutingPathOnSortFields() throws Exception { assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message")); } + public void testRoutingPathOnSortFieldsDisabledInOldNode() throws Exception { + var settings = Settings.builder() + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message") + .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) + .build(); + Settings result = generateLogsdbSettings(settings, null, Version.V_8_17_0); + assertTrue(result.isEmpty()); + } + public void testRoutingPathOnSortFieldsFilterTimestamp() throws Exception { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp") @@ -810,7 +837,7 @@ public void testExplicitRoutingPathMatchesSortFields() throws Exception { assertTrue(result.isEmpty()); } - public void testExplicitRoutingPathDoesNotMatchSortFields() throws Exception { + public void testExplicitRoutingPathDoesNotMatchSortFields() { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "host,message,foo") @@ -829,6 +856,22 @@ public void testExplicitRoutingPathDoesNotMatchSortFields() throws Exception { ); } + public void testExplicitRoutingPathNotAllowedByLicense() throws Exception { + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); + when(licenseState.isAllowed(same(LogsdbLicenseService.LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE))).thenReturn(false); + logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY); + logsdbLicenseService.setLicenseState(licenseState); + + var settings = Settings.builder() + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message") + .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) + .build(); + Settings result = generateLogsdbSettings(settings); + assertFalse(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.get(result)); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); + } + public void testSortAndHostNamePropagateValue() throws Exception { var settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) @@ -852,6 +895,42 @@ public void testSortAndHostNameWithCustomSortConfig() throws Exception { assertEquals(0, newMapperServiceCounter.get()); } + public void testSortAndHostNoHost() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNoHostOldNode() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); + assertTrue(result.isEmpty()); + } + public void testSortAndHostNameKeyword() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 7fa2f11880f40..e3faf3bdb5c1b 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.logsdb; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; @@ -25,7 +26,7 @@ import java.time.ZoneOffset; import java.util.List; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createGoldOrPlatinumLicense; +import static org.elasticsearch.xpack.logsdb.LogsdbLicenseServiceTests.createGoldOrPlatinumLicense; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,12 +41,12 @@ public void setup() throws Exception { License license = createGoldOrPlatinumLicense(); var licenseState = new XPackLicenseState(() -> time, new XPackLicenseStatus(license.operationMode(), true, null)); - var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + var licenseService = new LogsdbLicenseService(Settings.EMPTY); licenseService.setLicenseState(licenseState); var mockLicenseService = mock(LicenseService.class); when(mockLicenseService.getLicense()).thenReturn(license); - SyntheticSourceLicenseService syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + LogsdbLicenseService syntheticSourceLicenseService = new LogsdbLicenseService(Settings.EMPTY); syntheticSourceLicenseService.setLicenseState(licenseState); syntheticSourceLicenseService.setLicenseService(mockLicenseService); @@ -53,6 +54,8 @@ public void setup() throws Exception { provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), IndexVersion::current, + () -> Version.CURRENT, + true, true ); } @@ -102,12 +105,12 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception long time = LocalDateTime.of(2024, 12, 31, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); var licenseState = new XPackLicenseState(() -> time, new XPackLicenseStatus(license.operationMode(), true, null)); - var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + var licenseService = new LogsdbLicenseService(Settings.EMPTY); licenseService.setLicenseState(licenseState); var mockLicenseService = mock(LicenseService.class); when(mockLicenseService.getLicense()).thenReturn(license); - SyntheticSourceLicenseService syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + LogsdbLicenseService syntheticSourceLicenseService = new LogsdbLicenseService(Settings.EMPTY); syntheticSourceLicenseService.setLicenseState(licenseState); syntheticSourceLicenseService.setLicenseService(mockLicenseService); @@ -115,6 +118,8 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), IndexVersion::current, + () -> Version.CURRENT, + true, true ); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java similarity index 75% rename from x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java rename to x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java index 0eb0d21ff2e78..60701e9402e6c 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java @@ -26,23 +26,43 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class SyntheticSourceLicenseServiceTests extends ESTestCase { +public class LogsdbLicenseServiceTests extends ESTestCase { private LicenseService mockLicenseService; - private SyntheticSourceLicenseService licenseService; + private LogsdbLicenseService licenseService; @Before public void setup() throws Exception { mockLicenseService = mock(LicenseService.class); License license = createEnterpriseLicense(); when(mockLicenseService.getLicense()).thenReturn(license); - licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + licenseService = new LogsdbLicenseService(Settings.EMPTY); + } + + public void testAllowRoutingOnSortFields() { + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); + when(licenseState.isAllowed(same(LogsdbLicenseService.LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE))).thenReturn(true); + licenseService.setLicenseState(licenseState); + licenseService.setLicenseService(mockLicenseService); + assertTrue(licenseService.allowLogsdbRoutingOnSortField(false)); + Mockito.verify(licenseState, Mockito.times(1)).featureUsed(any()); + } + + public void testAllowRoutingOnSortFieldsTemplateValidation() { + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); + when(licenseState.isAllowed(same(LogsdbLicenseService.LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE))).thenReturn(true); + licenseService.setLicenseState(licenseState); + licenseService.setLicenseService(mockLicenseService); + assertTrue(licenseService.allowLogsdbRoutingOnSortField(true)); + Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); } public void testLicenseAllowsSyntheticSource() { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertFalse( @@ -55,7 +75,7 @@ public void testLicenseAllowsSyntheticSource() { public void testLicenseAllowsSyntheticSourceTemplateValidation() { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertFalse( @@ -65,10 +85,10 @@ public void testLicenseAllowsSyntheticSourceTemplateValidation() { Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); } - public void testDefaultDisallow() { + public void testDefaultDisallowSyntheticSource() { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertTrue( @@ -78,10 +98,10 @@ public void testDefaultDisallow() { Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); } - public void testFallback() { + public void testFallbackSyntheticSource() { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); licenseService.setSyntheticSourceFallback(true); @@ -101,15 +121,15 @@ public void testGoldOrPlatinumLicense() throws Exception { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertFalse( "legacy licensed usage is allowed, so not fallback to stored source", licenseService.fallbackToStoredSource(false, true) ); - Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE)); - Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY)); Mockito.verify(licenseState, Mockito.times(1)).featureUsed(any()); } @@ -121,7 +141,7 @@ public void testGoldOrPlatinumLicenseLegacyLicenseNotAllowed() throws Exception MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertTrue( @@ -129,7 +149,7 @@ public void testGoldOrPlatinumLicenseLegacyLicenseNotAllowed() throws Exception licenseService.fallbackToStoredSource(false, false) ); Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); - Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE)); } public void testGoldOrPlatinumLicenseBeyondCutoffDate() throws Exception { @@ -141,17 +161,17 @@ public void testGoldOrPlatinumLicenseBeyondCutoffDate() throws Exception { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertTrue("beyond cutoff date, so fallback to stored source", licenseService.fallbackToStoredSource(false, true)); Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); - Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE)); } public void testGoldOrPlatinumLicenseCustomCutoffDate() throws Exception { - licenseService = new SyntheticSourceLicenseService(Settings.EMPTY, "2025-01-02T00:00"); + licenseService = new LogsdbLicenseService(Settings.EMPTY, "2025-01-02T00:00"); long start = LocalDateTime.of(2025, 1, 3, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); License license = createGoldOrPlatinumLicense(start); @@ -161,12 +181,12 @@ public void testGoldOrPlatinumLicenseCustomCutoffDate() throws Exception { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); - when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); + when(licenseState.isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertTrue("custom cutoff date, so fallback to stored source", licenseService.fallbackToStoredSource(false, true)); - Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(LogsdbLicenseService.SYNTHETIC_SOURCE_FEATURE)); Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); } diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml index 5c5ed36168241..19f32d9067f38 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml @@ -754,7 +754,7 @@ modify logsdb index source mode to stored after index creation: _source: mode: stored - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" } + - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [null] to [stored]" } --- modify time_series index source mode to disabled after index creation: @@ -812,4 +812,4 @@ modify time_series index source mode to stored after index creation: _source: mode: stored - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" } + - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [null] to [stored]" } diff --git a/x-pack/plugin/logstash/build.gradle b/x-pack/plugin/logstash/build.gradle index 37bd972629aa2..69ab5b7f89c5e 100644 --- a/x-pack/plugin/logstash/build.gradle +++ b/x-pack/plugin/logstash/build.gradle @@ -1,5 +1,5 @@ apply plugin: 'elasticsearch.internal-es-plugin' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name = 'x-pack-logstash' @@ -17,10 +17,6 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation(project(path: xpackModule('core'))) javaRestTestImplementation(testArtifact(project(xpackModule('core')))) -} -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'true' - user username: 'x_pack_rest_user', password: 'x-pack-test-password' + clusterModules project(':modules:analysis-common') } diff --git a/x-pack/plugin/logstash/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/LogstashSystemIndexIT.java b/x-pack/plugin/logstash/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/LogstashSystemIndexIT.java index bf7f88d0fc86b..1ef4cbc3f2820 100644 --- a/x-pack/plugin/logstash/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/LogstashSystemIndexIT.java +++ b/x-pack/plugin/logstash/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/LogstashSystemIndexIT.java @@ -17,10 +17,12 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; import java.io.IOException; import java.util.ArrayList; @@ -31,6 +33,15 @@ import static org.hamcrest.Matchers.is; public class LogstashSystemIndexIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("x-pack-logstash") + .module("analysis-common") + .setting("xpack.security.enabled", "true") + .user("x_pack_rest_user", "x-pack-test-password") + .build(); + static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( "x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING @@ -41,6 +52,11 @@ protected Settings restClientSettings() { return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); } + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + public void testPipelineCRUD() throws Exception { // put pipeline final String pipelineJson = getPipelineJson(); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java index fea55e793d638..9320df583b4c5 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java @@ -17,7 +17,7 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.xpack.aggregatemetric.aggregations.metrics.AggregateMetricsAggregatorsRegistrar; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -32,7 +32,7 @@ public class AggregateMetricMapperPlugin extends Plugin implements MapperPlugin, @Override public Map getMappers() { - return singletonMap(AggregateDoubleMetricFieldMapper.CONTENT_TYPE, AggregateDoubleMetricFieldMapper.PARSER); + return singletonMap(AggregateMetricDoubleFieldMapper.CONTENT_TYPE, AggregateMetricDoubleFieldMapper.PARSER); } @Override diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregator.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregator.java index add4fb3e5d2db..a12d476af3ac1 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregator.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregator.java @@ -24,14 +24,14 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSource; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.Map; class AggregateMetricBackedAvgAggregator extends NumericMetricsAggregator.SingleValue { - final AggregateMetricsValuesSource.AggregateDoubleMetric valuesSource; + final AggregateMetricsValuesSource.AggregateMetricDouble valuesSource; LongArray counts; DoubleArray sums; @@ -47,7 +47,7 @@ class AggregateMetricBackedAvgAggregator extends NumericMetricsAggregator.Single ) throws IOException { super(name, context, parent, metadata); assert valuesSourceConfig.hasValues(); - this.valuesSource = (AggregateMetricsValuesSource.AggregateDoubleMetric) valuesSourceConfig.getValuesSource(); + this.valuesSource = (AggregateMetricsValuesSource.AggregateMetricDouble) valuesSourceConfig.getValuesSource(); final BigArrays bigArrays = context.bigArrays(); counts = bigArrays.newLongArray(1, true); sums = bigArrays.newDoubleArray(1, true); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregator.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregator.java index dd485ec218371..a007f334a69e2 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregator.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregator.java @@ -24,14 +24,14 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSource; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.Map; class AggregateMetricBackedMaxAggregator extends NumericMetricsAggregator.SingleValue { - private final AggregateMetricsValuesSource.AggregateDoubleMetric valuesSource; + private final AggregateMetricsValuesSource.AggregateMetricDouble valuesSource; final DocValueFormat formatter; DoubleArray maxes; @@ -44,7 +44,7 @@ class AggregateMetricBackedMaxAggregator extends NumericMetricsAggregator.Single ) throws IOException { super(name, context, parent, metadata); assert config.hasValues(); - this.valuesSource = (AggregateMetricsValuesSource.AggregateDoubleMetric) config.getValuesSource(); + this.valuesSource = (AggregateMetricsValuesSource.AggregateMetricDouble) config.getValuesSource(); maxes = context.bigArrays().newDoubleArray(1, false); maxes.fill(0, maxes.size(), Double.NEGATIVE_INFINITY); this.formatter = config.format(); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregator.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregator.java index 5a70801b8ae76..3b024c512aa82 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregator.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregator.java @@ -24,14 +24,14 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSource; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.Map; class AggregateMetricBackedMinAggregator extends NumericMetricsAggregator.SingleValue { - private final AggregateMetricsValuesSource.AggregateDoubleMetric valuesSource; + private final AggregateMetricsValuesSource.AggregateMetricDouble valuesSource; final DocValueFormat format; DoubleArray mins; @@ -44,7 +44,7 @@ class AggregateMetricBackedMinAggregator extends NumericMetricsAggregator.Single ) throws IOException { super(name, context, parent, metadata); assert config.hasValues(); - this.valuesSource = (AggregateMetricsValuesSource.AggregateDoubleMetric) config.getValuesSource(); + this.valuesSource = (AggregateMetricsValuesSource.AggregateMetricDouble) config.getValuesSource(); mins = context.bigArrays().newDoubleArray(1, false); mins.fill(0, mins.size(), Double.POSITIVE_INFINITY); this.format = config.format(); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregator.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregator.java index f4c28d7381214..480590b359bd3 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregator.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregator.java @@ -23,14 +23,14 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSource; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.Map; class AggregateMetricBackedSumAggregator extends NumericMetricsAggregator.SingleValue { - private final AggregateMetricsValuesSource.AggregateDoubleMetric valuesSource; + private final AggregateMetricsValuesSource.AggregateMetricDouble valuesSource; private final DocValueFormat format; private DoubleArray sums; @@ -45,7 +45,7 @@ class AggregateMetricBackedSumAggregator extends NumericMetricsAggregator.Single ) throws IOException { super(name, context, parent, metadata); assert valuesSourceConfig.hasValues(); - this.valuesSource = (AggregateMetricsValuesSource.AggregateDoubleMetric) valuesSourceConfig.getValuesSource(); + this.valuesSource = (AggregateMetricsValuesSource.AggregateMetricDouble) valuesSourceConfig.getValuesSource(); sums = context.bigArrays().newDoubleArray(1, true); compensations = context.bigArrays().newDoubleArray(1, true); this.format = valuesSourceConfig.format(); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregator.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregator.java index 065a5411b0bcb..49b3fd8846f9c 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregator.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregator.java @@ -20,7 +20,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSource; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; import java.io.IOException; import java.util.Map; @@ -32,7 +32,7 @@ */ class AggregateMetricBackedValueCountAggregator extends NumericMetricsAggregator.SingleValue { - private final AggregateMetricsValuesSource.AggregateDoubleMetric valuesSource; + private final AggregateMetricsValuesSource.AggregateMetricDouble valuesSource; // a count per bucket LongArray counts; @@ -46,7 +46,7 @@ class AggregateMetricBackedValueCountAggregator extends NumericMetricsAggregator ) throws IOException { super(name, aggregationContext, parent, metadata); assert valuesSourceConfig.hasValues(); - this.valuesSource = (AggregateMetricsValuesSource.AggregateDoubleMetric) valuesSourceConfig.getValuesSource(); + this.valuesSource = (AggregateMetricsValuesSource.AggregateMetricDouble) valuesSourceConfig.getValuesSource(); counts = bigArrays().newLongArray(1, true); } @@ -55,7 +55,7 @@ public LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, final BigArrays bigArrays = bigArrays(); final SortedNumericDoubleValues values = valuesSource.getAggregateMetricValues( aggCtx.getLeafReaderContext(), - AggregateDoubleMetricFieldMapper.Metric.value_count + AggregateMetricDoubleFieldMapper.Metric.value_count ); return new LeafBucketCollectorBase(sub, values) { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSource.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSource.java index a964573dbb5d9..c4a9c37fcf380 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSource.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSource.java @@ -13,23 +13,23 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.aggregations.AggregationErrors; import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.xpack.aggregatemetric.fielddata.IndexAggregateDoubleMetricFieldData; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.fielddata.IndexAggregateMetricDoubleFieldData; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.function.Function; public class AggregateMetricsValuesSource { - public abstract static class AggregateDoubleMetric extends org.elasticsearch.search.aggregations.support.ValuesSource { + public abstract static class AggregateMetricDouble extends org.elasticsearch.search.aggregations.support.ValuesSource { public abstract SortedNumericDoubleValues getAggregateMetricValues(LeafReaderContext context, Metric metric) throws IOException; - public static class Fielddata extends AggregateDoubleMetric { + public static class Fielddata extends AggregateMetricDouble { - protected final IndexAggregateDoubleMetricFieldData indexFieldData; + protected final IndexAggregateMetricDoubleFieldData indexFieldData; - public Fielddata(IndexAggregateDoubleMetricFieldData indexFieldData) { + public Fielddata(IndexAggregateMetricDoubleFieldData indexFieldData) { this.indexFieldData = indexFieldData; } @@ -51,7 +51,7 @@ public boolean advanceExact(int doc) throws IOException { @Override protected Function roundingPreparer(AggregationContext context) throws IOException { - throw AggregationErrors.unsupportedRounding(AggregateDoubleMetricFieldMapper.CONTENT_TYPE); + throw AggregationErrors.unsupportedRounding(AggregateMetricDoubleFieldMapper.CONTENT_TYPE); } public SortedNumericDoubleValues getAggregateMetricValues(LeafReaderContext context, Metric metric) throws IOException { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSourceType.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSourceType.java index bcac3f12fd131..e47275ed4b756 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSourceType.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/aggregations/support/AggregateMetricsValuesSourceType.java @@ -15,7 +15,7 @@ import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.fielddata.IndexAggregateDoubleMetricFieldData; +import org.elasticsearch.xpack.aggregatemetric.fielddata.IndexAggregateMetricDoubleFieldData; import java.util.Locale; import java.util.function.LongSupplier; @@ -43,7 +43,7 @@ public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType sc public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { final IndexFieldData indexFieldData = fieldContext.indexFieldData(); - if ((indexFieldData instanceof IndexAggregateDoubleMetricFieldData) == false) { + if ((indexFieldData instanceof IndexAggregateMetricDoubleFieldData) == false) { throw new IllegalArgumentException( "Expected aggregate_metric_double type on field [" + fieldContext.field() @@ -52,7 +52,7 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa + "]" ); } - return new AggregateMetricsValuesSource.AggregateDoubleMetric.Fielddata((IndexAggregateDoubleMetricFieldData) indexFieldData); + return new AggregateMetricsValuesSource.AggregateMetricDouble.Fielddata((IndexAggregateMetricDoubleFieldData) indexFieldData); } @Override diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/IndexAggregateDoubleMetricFieldData.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/IndexAggregateMetricDoubleFieldData.java similarity index 83% rename from x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/IndexAggregateDoubleMetricFieldData.java rename to x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/IndexAggregateMetricDoubleFieldData.java index eb07f9c641efb..a98b6eb4c04a4 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/IndexAggregateDoubleMetricFieldData.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/IndexAggregateMetricDoubleFieldData.java @@ -13,12 +13,12 @@ /** * Specialization of {@link IndexFieldData} for aggregate_metric. */ -public abstract class IndexAggregateDoubleMetricFieldData implements IndexFieldData { +public abstract class IndexAggregateMetricDoubleFieldData implements IndexFieldData { protected final String fieldName; protected final ValuesSourceType valuesSourceType; - public IndexAggregateDoubleMetricFieldData(String fieldName, ValuesSourceType valuesSourceType) { + public IndexAggregateMetricDoubleFieldData(String fieldName, ValuesSourceType valuesSourceType) { this.fieldName = fieldName; this.valuesSourceType = valuesSourceType; } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/LeafAggregateDoubleMetricFieldData.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/LeafAggregateMetricDoubleFieldData.java similarity index 72% rename from x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/LeafAggregateDoubleMetricFieldData.java rename to x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/LeafAggregateMetricDoubleFieldData.java index c8a89456be5e5..c11ccd5d4ff2e 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/LeafAggregateDoubleMetricFieldData.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/fielddata/LeafAggregateMetricDoubleFieldData.java @@ -8,12 +8,12 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; /** - * {@link LeafFieldData} specialization for aggregate_double_metric data. + * {@link LeafFieldData} specialization for aggregate_metric_double data. */ -public interface LeafAggregateDoubleMetricFieldData extends LeafFieldData { +public interface LeafAggregateMetricDoubleFieldData extends LeafFieldData { /** * Return aggregate_metric of double values for a given metric diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java similarity index 96% rename from x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java rename to x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java index a58f8dae8cc73..3ab49126ecb0d 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java @@ -59,8 +59,8 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentSubParser; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.fielddata.IndexAggregateDoubleMetricFieldData; -import org.elasticsearch.xpack.aggregatemetric.fielddata.LeafAggregateDoubleMetricFieldData; +import org.elasticsearch.xpack.aggregatemetric.fielddata.IndexAggregateMetricDoubleFieldData; +import org.elasticsearch.xpack.aggregatemetric.fielddata.LeafAggregateMetricDoubleFieldData; import java.io.IOException; import java.time.ZoneId; @@ -78,15 +78,15 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; /** A {@link FieldMapper} for a field containing aggregate metrics such as min/max/value_count etc. */ -public class AggregateDoubleMetricFieldMapper extends FieldMapper { +public class AggregateMetricDoubleFieldMapper extends FieldMapper { - private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(AggregateDoubleMetricFieldMapper.class); + private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(AggregateMetricDoubleFieldMapper.class); public static final String CONTENT_TYPE = "aggregate_metric_double"; public static final String SUBFIELD_SEPARATOR = "."; - private static AggregateDoubleMetricFieldMapper toType(FieldMapper in) { - return (AggregateDoubleMetricFieldMapper) in; + private static AggregateMetricDoubleFieldMapper toType(FieldMapper in) { + return (AggregateMetricDoubleFieldMapper) in; } /** @@ -97,7 +97,7 @@ private static AggregateDoubleMetricFieldMapper toType(FieldMapper in) { * @return the name of the subfield */ public static String subfieldName(String fieldName, Metric metric) { - return fieldName + AggregateDoubleMetricFieldMapper.SUBFIELD_SEPARATOR + metric.name(); + return fieldName + AggregateMetricDoubleFieldMapper.SUBFIELD_SEPARATOR + metric.name(); } /** @@ -150,7 +150,7 @@ public static final class Builder extends FieldMapper.Builder { /** * Parameter that marks this field as a time series metric defining its time series metric type. - * For {@link AggregateDoubleMetricFieldMapper} fields gauge, counter and summary metric types are + * For {@link AggregateMetricDoubleFieldMapper} fields gauge, counter and summary metric types are * supported. */ private final Parameter timeSeriesMetric; @@ -194,7 +194,7 @@ public Builder metric(MetricType metric) { } @Override - public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) { + public AggregateMetricDoubleFieldMapper build(MapperBuilderContext context) { if (multiFieldsBuilder.hasMultiFields()) { DEPRECATION_LOGGER.warn( DeprecationCategory.MAPPINGS, @@ -261,7 +261,7 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) { throw new IllegalArgumentException("Duplicate keys " + l + "and " + r + "."); }, () -> new EnumMap<>(Metric.class))); - AggregateDoubleMetricFieldType metricFieldType = new AggregateDoubleMetricFieldType( + AggregateMetricDoubleFieldType metricFieldType = new AggregateMetricDoubleFieldType( context.buildFullName(leafName()), meta.getValue(), timeSeriesMetric.getValue() @@ -269,7 +269,7 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) { metricFieldType.setMetricFields(metricFields); metricFieldType.setDefaultMetric(defaultMetric.getValue()); - return new AggregateDoubleMetricFieldMapper(leafName(), metricFieldType, metricMappers, builderParams(this, context), this); + return new AggregateMetricDoubleFieldMapper(leafName(), metricFieldType, metricMappers, builderParams(this, context), this); } } @@ -278,7 +278,7 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) { notInMultiFields(CONTENT_TYPE) ); - public static final class AggregateDoubleMetricFieldType extends SimpleMappedFieldType { + public static final class AggregateMetricDoubleFieldType extends SimpleMappedFieldType { private EnumMap metricFields; @@ -286,11 +286,11 @@ public static final class AggregateDoubleMetricFieldType extends SimpleMappedFie private final MetricType metricType; - public AggregateDoubleMetricFieldType(String name) { + public AggregateMetricDoubleFieldType(String name) { this(name, Collections.emptyMap(), null); } - public AggregateDoubleMetricFieldType(String name, Map meta, MetricType metricType) { + public AggregateMetricDoubleFieldType(String name, Map meta, MetricType metricType) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); this.metricType = metricType; } @@ -326,7 +326,7 @@ public Map getMetricFields() { public void addMetricField(Metric m, NumberFieldMapper.NumberFieldType subfield) { if (metricFields == null) { - metricFields = new EnumMap<>(AggregateDoubleMetricFieldMapper.Metric.class); + metricFields = new EnumMap<>(AggregateMetricDoubleFieldMapper.Metric.class); } if (name() == null) { @@ -408,13 +408,13 @@ public boolean isAggregatable() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - return (cache, breakerService) -> new IndexAggregateDoubleMetricFieldData( + return (cache, breakerService) -> new IndexAggregateMetricDoubleFieldData( name(), AggregateMetricsValuesSourceType.AGGREGATE_METRIC ) { @Override - public LeafAggregateDoubleMetricFieldData load(LeafReaderContext context) { - return new LeafAggregateDoubleMetricFieldData() { + public LeafAggregateMetricDoubleFieldData load(LeafReaderContext context) { + return new LeafAggregateMetricDoubleFieldData() { @Override public SortedNumericDoubleValues getAggregateMetricValues(final Metric metric) { try { @@ -476,7 +476,7 @@ public long ramBytesUsed() { } @Override - public LeafAggregateDoubleMetricFieldData loadDirect(LeafReaderContext context) { + public LeafAggregateMetricDoubleFieldData loadDirect(LeafReaderContext context) { return load(context); } @@ -677,7 +677,7 @@ public MetricType getMetricType() { private final IndexMode indexMode; - private AggregateDoubleMetricFieldMapper( + private AggregateMetricDoubleFieldMapper( String simpleName, MappedFieldType mappedFieldType, EnumMap metricFieldMappers, @@ -705,8 +705,8 @@ Metric defaultMetric() { } @Override - public AggregateDoubleMetricFieldType fieldType() { - return (AggregateDoubleMetricFieldType) super.fieldType(); + public AggregateMetricDoubleFieldType fieldType() { + return (AggregateMetricDoubleFieldType) super.fieldType(); } @Override diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregatorTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregatorTests.java index 8378f99b2d7b2..fade3f68376d0 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregatorTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedAvgAggregatorTests.java @@ -27,15 +27,15 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.subfieldName; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.subfieldName; public class AggregateMetricBackedAvgAggregatorTests extends AggregatorTestCase { @@ -116,8 +116,8 @@ public void testQueryFiltering() throws IOException { * @param fieldName the name of the field * @return the created field type */ - private AggregateDoubleMetricFieldType createDefaultFieldType(String fieldName) { - AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(fieldName); + private AggregateMetricDoubleFieldType createDefaultFieldType(String fieldName) { + AggregateMetricDoubleFieldType fieldType = new AggregateMetricDoubleFieldType(fieldName); for (Metric m : List.of(Metric.value_count, Metric.sum)) { String subfieldName = subfieldName(fieldName, m); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregatorTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregatorTests.java index 9cbafff116b4c..33e9151773fc2 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregatorTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMaxAggregatorTests.java @@ -27,15 +27,15 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.subfieldName; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.subfieldName; public class AggregateMetricBackedMaxAggregatorTests extends AggregatorTestCase { @@ -116,8 +116,8 @@ public void testQueryFiltering() throws IOException { * @param fieldName the name of the field * @return the created field type */ - private AggregateDoubleMetricFieldType createDefaultFieldType(String fieldName) { - AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(fieldName); + private AggregateMetricDoubleFieldType createDefaultFieldType(String fieldName) { + AggregateMetricDoubleFieldType fieldType = new AggregateMetricDoubleFieldType(fieldName); for (Metric m : List.of(Metric.min, Metric.max)) { String subfieldName = subfieldName(fieldName, m); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregatorTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregatorTests.java index fb4ea5785fbce..0f655b90a2358 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregatorTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedMinAggregatorTests.java @@ -27,15 +27,15 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.subfieldName; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.subfieldName; public class AggregateMetricBackedMinAggregatorTests extends AggregatorTestCase { @@ -116,8 +116,8 @@ public void testQueryFiltering() throws IOException { * @param fieldName the name of the field * @return the created field type */ - private AggregateDoubleMetricFieldType createDefaultFieldType(String fieldName) { - AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(fieldName); + private AggregateMetricDoubleFieldType createDefaultFieldType(String fieldName) { + AggregateMetricDoubleFieldType fieldType = new AggregateMetricDoubleFieldType(fieldName); for (Metric m : List.of(Metric.min, Metric.max)) { String subfieldName = subfieldName(fieldName, m); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregatorTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregatorTests.java index 91a34b4643456..e0e421189497c 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregatorTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedSumAggregatorTests.java @@ -27,15 +27,15 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.subfieldName; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.subfieldName; public class AggregateMetricBackedSumAggregatorTests extends AggregatorTestCase { @@ -116,8 +116,8 @@ public void testQueryFiltering() throws IOException { * @param fieldName the name of the field * @return the created field type */ - private AggregateDoubleMetricFieldType createDefaultFieldType(String fieldName) { - AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(fieldName); + private AggregateMetricDoubleFieldType createDefaultFieldType(String fieldName) { + AggregateMetricDoubleFieldType fieldType = new AggregateMetricDoubleFieldType(fieldName); for (Metric m : List.of(Metric.value_count, Metric.sum)) { String subfieldName = subfieldName(fieldName, m); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregatorTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregatorTests.java index faff3c2d7cb30..dbae604b8f725 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregatorTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/aggregations/metrics/AggregateMetricBackedValueCountAggregatorTests.java @@ -27,15 +27,15 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.aggregatemetric.aggregations.support.AggregateMetricsValuesSourceType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.List; import java.util.function.Consumer; import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.subfieldName; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.subfieldName; public class AggregateMetricBackedValueCountAggregatorTests extends AggregatorTestCase { @@ -115,8 +115,8 @@ public void testQueryFiltering() throws IOException { * @param fieldName the name of the field * @return the created field type */ - private AggregateDoubleMetricFieldType createDefaultFieldType(String fieldName) { - AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(fieldName); + private AggregateMetricDoubleFieldType createDefaultFieldType(String fieldName) { + AggregateMetricDoubleFieldType fieldType = new AggregateMetricDoubleFieldType(fieldName); for (Metric m : List.of(Metric.value_count, Metric.sum)) { String subfieldName = subfieldName(fieldName, m); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java similarity index 94% rename from x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java rename to x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java index 0d62e7a9c1fd2..3674043a72766 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import org.hamcrest.Matchers; import org.junit.AssumptionViolatedException; @@ -38,18 +38,18 @@ import java.util.Map; import java.util.function.Function; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.IGNORE_MALFORMED; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.METRICS; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Names.IGNORE_MALFORMED; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Names.METRICS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.core.IsInstanceOf.instanceOf; -public class AggregateDoubleMetricFieldMapperTests extends MapperTestCase { +public class AggregateMetricDoubleFieldMapperTests extends MapperTestCase { public static final String METRICS_FIELD = METRICS; - public static final String CONTENT_TYPE = AggregateDoubleMetricFieldMapper.CONTENT_TYPE; - public static final String DEFAULT_METRIC = AggregateDoubleMetricFieldMapper.Names.DEFAULT_METRIC; + public static final String CONTENT_TYPE = AggregateMetricDoubleFieldMapper.CONTENT_TYPE; + public static final String DEFAULT_METRIC = AggregateMetricDoubleFieldMapper.Names.DEFAULT_METRIC; @Override protected Collection getPlugins() { @@ -109,7 +109,7 @@ public void testParseValue() throws Exception { assertEquals("DoubleField ", doc.rootDoc().getField("field.min").toString()); Mapper fieldMapper = mapper.mappers().getMapper("field"); - assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + assertThat(fieldMapper, instanceOf(AggregateMetricDoubleFieldMapper.class)); } /** @@ -325,8 +325,8 @@ public void testExplicitDefaultMetric() throws Exception { ); Mapper fieldMapper = mapper.mappers().getMapper("field"); - assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); - assertEquals(Metric.sum, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric()); + assertThat(fieldMapper, instanceOf(AggregateMetricDoubleFieldMapper.class)); + assertEquals(Metric.sum, ((AggregateMetricDoubleFieldMapper) fieldMapper).defaultMetric()); } /** @@ -338,8 +338,8 @@ public void testImplicitDefaultMetricSingleMetric() throws Exception { ); Mapper fieldMapper = mapper.mappers().getMapper("field"); - assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); - assertEquals(Metric.value_count, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric); + assertThat(fieldMapper, instanceOf(AggregateMetricDoubleFieldMapper.class)); + assertEquals(Metric.value_count, ((AggregateMetricDoubleFieldMapper) fieldMapper).defaultMetric); } /** @@ -348,8 +348,8 @@ public void testImplicitDefaultMetricSingleMetric() throws Exception { public void testImplicitDefaultMetric() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); Mapper fieldMapper = mapper.mappers().getMapper("field"); - assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); - assertEquals(Metric.max, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric); + assertThat(fieldMapper, instanceOf(AggregateMetricDoubleFieldMapper.class)); + assertEquals(Metric.max, ((AggregateMetricDoubleFieldMapper) fieldMapper).defaultMetric); } /** @@ -418,7 +418,7 @@ public void testParseNestedValue() throws Exception { ); Mapper fieldMapper = mapper.mappers().getMapper("field.subfield"); - assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + assertThat(fieldMapper, instanceOf(AggregateMetricDoubleFieldMapper.class)); ParsedDocument doc = mapper.parse( source( b -> b.startObject("field") @@ -462,7 +462,7 @@ public void testFieldCaps() throws IOException { protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) { assertThat(query, Matchers.instanceOf(FieldExistsQuery.class)); FieldExistsQuery fieldExistsQuery = (FieldExistsQuery) query; - String defaultMetric = ((AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType) fieldType).getDefaultMetric().name(); + String defaultMetric = ((AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType) fieldType).getDefaultMetric().name(); assertEquals("field." + defaultMetric, fieldExistsQuery.getField()); assertNoFieldNamesField(fields); } @@ -488,10 +488,10 @@ public void testCannotBeUsedInMultifields() { public void testMetricType() throws IOException { // Test default setting MapperService mapperService = createMapperService(fieldMapping(b -> minimalMapping(b))); - AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType ft = - (AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType) mapperService.fieldType("field"); + AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType ft = + (AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType) mapperService.fieldType("field"); assertNull(ft.getMetricType()); - assertMetricType("gauge", AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType::getMetricType); + assertMetricType("gauge", AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType::getMetricType); { // Test invalid metric type for this field type @@ -519,7 +519,7 @@ public void testMetricType() throws IOException { @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - return new AggregateDoubleMetricSyntheticSourceSupport(ignoreMalformed); + return new AggregateMetricDoubleSyntheticSourceSupport(ignoreMalformed); } @Override @@ -564,11 +564,11 @@ public void testArrayValueSyntheticSource() throws Exception { assertEquals(Strings.toString(expected), syntheticSource); } - protected final class AggregateDoubleMetricSyntheticSourceSupport implements SyntheticSourceSupport { + protected final class AggregateMetricDoubleSyntheticSourceSupport implements SyntheticSourceSupport { private final boolean malformedExample; private final EnumSet storedMetrics; - public AggregateDoubleMetricSyntheticSourceSupport(boolean malformedExample) { + public AggregateMetricDoubleSyntheticSourceSupport(boolean malformedExample) { this.malformedExample = malformedExample; this.storedMetrics = EnumSet.copyOf(randomNonEmptySubsetOf(Arrays.asList(Metric.values()))); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java similarity index 91% rename from x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java rename to x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java index 89c2799d8327d..55ecfc13b1f3e 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldTypeTests.java @@ -27,8 +27,8 @@ import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; -import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Metric; import java.io.IOException; import java.util.Collections; @@ -36,20 +36,20 @@ import java.util.Map; import static java.util.Arrays.asList; -import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.subfieldName; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.subfieldName; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class AggregateDoubleMetricFieldTypeTests extends FieldTypeTestCase { +public class AggregateMetricDoubleFieldTypeTests extends FieldTypeTestCase { - protected AggregateDoubleMetricFieldType createDefaultFieldType(String name, Map meta, Metric defaultMetric) { - AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(name, meta, null); - for (AggregateDoubleMetricFieldMapper.Metric m : List.of( - AggregateDoubleMetricFieldMapper.Metric.min, - AggregateDoubleMetricFieldMapper.Metric.max + protected AggregateMetricDoubleFieldType createDefaultFieldType(String name, Map meta, Metric defaultMetric) { + AggregateMetricDoubleFieldType fieldType = new AggregateMetricDoubleFieldType(name, meta, null); + for (AggregateMetricDoubleFieldMapper.Metric m : List.of( + AggregateMetricDoubleFieldMapper.Metric.min, + AggregateMetricDoubleFieldMapper.Metric.max )) { String subfieldName = subfieldName(fieldType.name(), m); NumberFieldMapper.NumberFieldType subfield = new NumberFieldMapper.NumberFieldType( diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle index 0a6fce7fee14e..d2cb60aceb755 100644 --- a/x-pack/plugin/mapper-constant-keyword/build.gradle +++ b/x-pack/plugin/mapper-constant-keyword/build.gradle @@ -18,7 +18,7 @@ dependencies { compileOnly project(path: xpackModule('core')) } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.getSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/migrate/build.gradle b/x-pack/plugin/migrate/build.gradle index 283362a637e78..80d2162dfff33 100644 --- a/x-pack/plugin/migrate/build.gradle +++ b/x-pack/plugin/migrate/build.gradle @@ -17,8 +17,15 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(xpackModule('ccr')) + testImplementation project(xpackModule('frozen-indices')) + testImplementation project(xpackModule('ilm')) testImplementation project(':modules:data-streams') testImplementation project(path: ':modules:reindex') + testImplementation project(path: ':modules:ingest-common') + testImplementation project(path: ':modules:lang-painless') + + internalClusterTestImplementation project(path: ':modules:lang-painless') + internalClusterTestImplementation project(path: ':modules:lang-painless:spi') } addQaCheckDependencies(project) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/AbstractFeatureMigrationIntegTest.java similarity index 67% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/AbstractFeatureMigrationIntegTest.java index 87126aa7a0f59..06981a2404475 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/AbstractFeatureMigrationIntegTest.java @@ -1,22 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; +import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.ActionFilter; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -28,13 +29,20 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.AssociatedIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.ingest.common.IngestCommonPlugin; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.migrate.MigratePlugin; import org.junit.Assert; import org.junit.Before; @@ -46,14 +54,27 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Function; - +import java.util.stream.Collectors; + +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; +import static java.util.Collections.unmodifiableSet; +import static org.elasticsearch.common.util.set.Sets.newHashSet; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, autoManageMasterNodes = false) public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase { @@ -69,9 +90,9 @@ public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase static final int INDEX_DOC_COUNT = 100; // arbitrarily chosen static final int INTERNAL_MANAGED_FLAG_VALUE = 1; static final String FIELD_NAME = "some_field"; - public static final Version NEEDS_UPGRADE_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.previousMajor(); + public static final Version NEEDS_UPGRADE_VERSION = SystemIndices.NO_UPGRADE_REQUIRED_VERSION.previousMajor(); public static final IndexVersion NEEDS_UPGRADE_INDEX_VERSION = IndexVersionUtils.getPreviousMajorVersion( - TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_INDEX_VERSION + SystemIndices.NO_UPGRADE_REQUIRED_INDEX_VERSION ); static final SystemIndexDescriptor EXTERNAL_UNMANAGED = SystemIndexDescriptor.builder() @@ -137,6 +158,18 @@ public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase protected String masterAndDataNode; protected String masterName; + protected static Metadata assertMetadataAfterMigration(String featureName) { + Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata(); + // Check that the results metadata is what we expect. + FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE); + assertThat(currentResults, notNullValue()); + assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(featureName))); + assertThat(currentResults.getFeatureStatuses().get(featureName).succeeded(), is(true)); + assertThat(currentResults.getFeatureStatuses().get(featureName).getFailedResourceName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(featureName).getException(), nullValue()); + return finalMetadata; + } + @Before public void setup() { assumeTrue( @@ -153,11 +186,21 @@ public void setup() { testPlugin.postMigrationHook.set((state, metadata) -> {}); } - public T getPlugin(Class type) { + protected T getPlugin(Class type) { final PluginsService pluginsService = internalCluster().getCurrentMasterNodeInstance(PluginsService.class); return pluginsService.filterPlugins(type).findFirst().get(); } + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(MigratePlugin.class); + plugins.add(ReindexPlugin.class); + plugins.add(TestPlugin.class); + plugins.add(IngestCommonPlugin.class); + return plugins; + } + protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) { assertThat( "the strategy used below to create index names for descriptors without a primary index name only works for simple patterns", @@ -174,7 +217,7 @@ protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) createRequest.setSettings( createSettings( NEEDS_UPGRADE_INDEX_VERSION, - descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE + descriptor.isExternal() ? EXTERNAL_UNMANAGED_FLAG_VALUE : INTERNAL_UNMANAGED_FLAG_VALUE ) ); } else { @@ -187,7 +230,7 @@ protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) ); } if (descriptor.isAutomaticallyManaged() == false) { - createRequest.setMapping(createMapping(false, descriptor.isInternal())); + createRequest.setMapping(createMapping(false, descriptor.isExternal() == false)); } CreateIndexResponse response = createRequest.get(); Assert.assertTrue(response.isShardsAcknowledged()); @@ -267,12 +310,60 @@ protected void assertIndexHasCorrectProperties( assertThat(thisIndexStats.getTotal().getDocs().getCount(), is((long) INDEX_DOC_COUNT)); } - public static class TestPlugin extends Plugin implements SystemIndexPlugin { + protected void executeMigration(String featureName) throws Exception { + startMigration(featureName); + + GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); + // The feature upgrade may take longer than ten seconds when tests are running + // in parallel, so we give assertBusy a thirty-second timeout. + assertBusy(() -> { + GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) + .get(); + logger.info(Strings.toString(statusResponse)); + assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); + }, 30, TimeUnit.SECONDS); + } + + protected static void startMigration(String featureName) throws InterruptedException, ExecutionException { + PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT); + PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); + assertThat(migrationResponse.getReason(), nullValue()); + assertThat(migrationResponse.getElasticsearchException(), nullValue()); + final Set migratingFeatures = migrationResponse.getFeatures() + .stream() + .map(PostFeatureUpgradeResponse.Feature::getFeatureName) + .collect(Collectors.toSet()); + assertThat(migratingFeatures, hasItem(featureName)); + } + + protected static TestPlugin.BlockingActionFilter blockAction(String actionTypeName) { + // Block the alias request to simulate a failure + InternalTestCluster internalTestCluster = internalCluster(); + ActionFilters actionFilters = internalTestCluster.getInstance(ActionFilters.class, internalTestCluster.getMasterName()); + TestPlugin.BlockingActionFilter blockingActionFilter = null; + for (ActionFilter filter : actionFilters.filters()) { + if (filter instanceof TestPlugin.BlockingActionFilter) { + blockingActionFilter = (TestPlugin.BlockingActionFilter) filter; + break; + } + } + assertNotNull("BlockingActionFilter should exist", blockingActionFilter); + blockingActionFilter.blockActions(actionTypeName); + return blockingActionFilter; + } + + public static class TestPlugin extends Plugin implements SystemIndexPlugin, ActionPlugin { public final AtomicReference>> preMigrationHook = new AtomicReference<>(); public final AtomicReference>> postMigrationHook = new AtomicReference<>(); + private final BlockingActionFilter blockingActionFilter; public TestPlugin() { + blockingActionFilter = new BlockingActionFilter(); + } + @Override + public List getActionFilters() { + return singletonList(blockingActionFilter); } @Override @@ -311,5 +402,30 @@ public void indicesMigrationComplete( postMigrationHook.get().accept(clusterService.state(), preUpgradeMetadata); listener.onResponse(true); } + + public static class BlockingActionFilter extends org.elasticsearch.action.support.ActionFilter.Simple { + private Set blockedActions = emptySet(); + + @Override + protected boolean apply(String action, ActionRequest request, ActionListener listener) { + if (blockedActions.contains(action)) { + throw new ElasticsearchException("force exception on [" + action + "]"); + } + return true; + } + + @Override + public int order() { + return 0; + } + + public void unblockAllActions() { + blockedActions = emptySet(); + } + + public void blockActions(String... actions) { + blockedActions = unmodifiableSet(newHashSet(actions)); + } + } } } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/FeatureMigrationIT.java similarity index 86% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/FeatureMigrationIT.java index cd5857da078ba..558cee075797f 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/FeatureMigrationIT.java @@ -1,22 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeResponse; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; @@ -39,9 +32,9 @@ import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.reindex.ReindexPlugin; -import org.elasticsearch.upgrades.FeatureMigrationResults; -import org.elasticsearch.upgrades.SingleFeatureMigrationResult; +import org.elasticsearch.system_indices.action.AbstractFeatureMigrationIntegTest.TestPlugin.BlockingActionFilter; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.SingleFeatureMigrationResult; import java.util.ArrayList; import java.util.Arrays; @@ -56,14 +49,11 @@ import java.util.stream.Collectors; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class FeatureMigrationIT extends AbstractFeatureMigrationIntegTest { @@ -102,9 +92,7 @@ protected boolean forbidPrivateIndexSettings() { @Override protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); - plugins.add(TestPlugin.class); plugins.add(SecondTestPlugin.class); - plugins.add(ReindexPlugin.class); plugins.add(PainlessPlugin.class); return plugins; } @@ -243,18 +231,6 @@ public void testMigrateSystemIndex() throws Exception { ); } - private static Metadata assertMetadataAfterMigration(String featureName) { - Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata(); - // Check that the results metadata is what we expect. - FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE); - assertThat(currentResults, notNullValue()); - assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(featureName))); - assertThat(currentResults.getFeatureStatuses().get(featureName).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(featureName).getFailedIndexName(), nullValue()); - assertThat(currentResults.getFeatureStatuses().get(featureName).getException(), nullValue()); - return finalMetadata; - } - public void testMigrateIndexWithWriteBlock() throws Exception { createSystemIndexForDescriptor(INTERNAL_UNMANAGED); @@ -274,6 +250,48 @@ public void testMigrateIndexWithWriteBlock() throws Exception { }); } + public void testIndexBlockIsRemovedWhenAliasRequestFails() throws Exception { + createSystemIndexForDescriptor(INTERNAL_UNMANAGED); + ensureGreen(); + + BlockingActionFilter blockingActionFilter = blockAction(TransportIndicesAliasesAction.NAME); + + // Start the migration + client().execute(PostFeatureUpgradeAction.INSTANCE, new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT)).get(); + + // Wait till the migration fails + assertBusy(() -> { + GetFeatureUpgradeStatusResponse statusResp = client().execute( + GetFeatureUpgradeStatusAction.INSTANCE, + new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT) + ).get(); + logger.info(Strings.toString(statusResp)); + assertThat(statusResp.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR)); + }); + + // Get the settings to see if the write block was removed + var allsettings = client().admin().indices().prepareGetSettings(INTERNAL_UNMANAGED.getIndexPattern()).get().getIndexToSettings(); + var internalUnmanagedOldIndexSettings = allsettings.get(".int-unman-old"); + var writeBlock = internalUnmanagedOldIndexSettings.get(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey()); + assertThat("Write block on old index should be removed on migration ERROR status", writeBlock, equalTo("false")); + + // Unblock the alias request + blockingActionFilter.unblockAllActions(); + + // Retry the migration + client().execute(PostFeatureUpgradeAction.INSTANCE, new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT)).get(); + + // Ensure that the migration is successful after the alias request is unblocked + assertBusy(() -> { + GetFeatureUpgradeStatusResponse statusResp = client().execute( + GetFeatureUpgradeStatusAction.INSTANCE, + new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT) + ).get(); + logger.info(Strings.toString(statusResp)); + assertThat(statusResp.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); + }); + } + public void testMigrationWillRunAfterError() throws Exception { createSystemIndexForDescriptor(INTERNAL_MANAGED); @@ -332,28 +350,6 @@ public void onFailure(Exception e) { }); } - private void executeMigration(String featureName) throws Exception { - PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT); - PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get(); - assertThat(migrationResponse.getReason(), nullValue()); - assertThat(migrationResponse.getElasticsearchException(), nullValue()); - final Set migratingFeatures = migrationResponse.getFeatures() - .stream() - .map(PostFeatureUpgradeResponse.Feature::getFeatureName) - .collect(Collectors.toSet()); - assertThat(migratingFeatures, hasItem(featureName)); - - GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); - // The feature upgrade may take longer than ten seconds when tests are running - // in parallel, so we give assertBusy a sixty-second timeout. - assertBusy(() -> { - GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) - .get(); - logger.info(Strings.toString(statusResponse)); - assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED)); - }, 60, TimeUnit.SECONDS); - } - public void testMigrateUsingScript() throws Exception { createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT); diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/MultiFeatureMigrationIT.java similarity index 91% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/MultiFeatureMigrationIT.java index bcce51fe97545..e15002c5b14ff 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/MultiFeatureMigrationIT.java @@ -1,23 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; @@ -28,8 +20,7 @@ import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.reindex.ReindexPlugin; -import org.elasticsearch.upgrades.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; import java.util.ArrayList; import java.util.Arrays; @@ -72,9 +63,7 @@ protected boolean forbidPrivateIndexSettings() { @Override protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); - plugins.add(TestPlugin.class); plugins.add(SecondPlugin.class); - plugins.add(ReindexPlugin.class); return plugins; } @@ -149,7 +138,7 @@ public void testMultipleFeatureMigration() throws Exception { assertThat(currentResults, notNullValue()); assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME))); assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedResourceName(), nullValue()); assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue()); secondPluginPreMigrationHookCalled.set(true); @@ -170,7 +159,7 @@ public void testMultipleFeatureMigration() throws Exception { assertThat(currentResults, notNullValue()); assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME))); assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedResourceName(), nullValue()); assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue()); secondPluginPostMigrationHookCalled.set(true); @@ -210,10 +199,10 @@ public void testMultipleFeatureMigration() throws Exception { assertThat(currentResults, notNullValue()); assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(2), hasKey(FEATURE_NAME), hasKey(SECOND_FEATURE_NAME))); assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedResourceName(), nullValue()); assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue()); assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).succeeded(), is(true)); - assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).getFailedIndexName(), nullValue()); + assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).getFailedResourceName(), nullValue()); assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).getException(), nullValue()); // Finally, verify that all the indices exist and have the properties we expect. diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemDataStreamMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemDataStreamMigrationIT.java new file mode 100644 index 0000000000000..8f0647a64f70d --- /dev/null +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemDataStreamMigrationIT.java @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.system_indices.action; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.After; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; + +public class SystemDataStreamMigrationIT extends AbstractFeatureMigrationIntegTest { + private static final String TEST_DATA_STREAM_NAME = ".test-data-stream"; + private static final String DATA_STREAM_FEATURE = "ds-feature"; + private static volatile SystemDataStreamDescriptor systemDataStreamDescriptor = createSystemDataStreamDescriptor( + NEEDS_UPGRADE_INDEX_VERSION + ); + + private static SystemDataStreamDescriptor createSystemDataStreamDescriptor(IndexVersion indexVersion) { + return new SystemDataStreamDescriptor( + TEST_DATA_STREAM_NAME, + "system data stream test", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder() + .template( + Template.builder() + .dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(true)) + .settings(indexSettings(indexVersion, 1, 0)) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(), + Map.of(), + List.of("product"), + ORIGIN, + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build(); + } + + @Override + protected boolean forbidPrivateIndexSettings() { + // We need to be able to set the index creation version manually. + return false; + } + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(DataStreamsPlugin.class); + plugins.add(DataStreamTestPlugin.class); + return plugins; + } + + @After + public void restoreDescriptor() { + // we need to do it in after, because we need to have systemDataStreamDescriptor in a correct state + // before next super.setup() is called + systemDataStreamDescriptor = createSystemDataStreamDescriptor(NEEDS_UPGRADE_INDEX_VERSION); + } + + private static void indexDocsToDataStream(String dataStreamName) { + BulkRequestBuilder bulkBuilder = client().prepareBulk(); + for (int i = 0; i < INDEX_DOC_COUNT; i++) { + IndexRequestBuilder requestBuilder = ESIntegTestCase.prepareIndex(dataStreamName) + .setId(Integer.toString(i)) + .setRequireDataStream(true) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DataStream.TIMESTAMP_FIELD_NAME, 1741271969000L, FIELD_NAME, "words words"); + bulkBuilder.add(requestBuilder); + } + + BulkResponse actionGet = bulkBuilder.get(); + assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false)); + } + + public void testMigrateSystemDataStream() throws Exception { + createDataStream(); + + indexDocsToDataStream(TEST_DATA_STREAM_NAME); + + simulateClusterUpgrade(); + + executeMigration(DATA_STREAM_FEATURE); + + // Waiting for shards to stabilize if indices were moved around + ensureGreen(); + + Metadata finalMetadata = assertMetadataAfterMigration(DATA_STREAM_FEATURE); + + DataStream dataStream = finalMetadata.dataStreams().get(TEST_DATA_STREAM_NAME); + assertNotNull(dataStream); + assertThat(dataStream.isSystem(), is(true)); + List backingIndices = dataStream.getIndices(); + assertThat(backingIndices, hasSize(2)); + for (Index backingIndex : backingIndices) { + IndexMetadata indexMetadata = finalMetadata.index(backingIndex); + assertThat(indexMetadata.isSystem(), is(true)); + assertThat(indexMetadata.getCreationVersion(), is(IndexVersion.current())); + } + } + + public void testMigrationRestartAfterFailure() throws Exception { + createDataStream(); + + indexDocsToDataStream(TEST_DATA_STREAM_NAME); + + simulateClusterUpgrade(); + + TestPlugin.BlockingActionFilter blockingActionFilter = blockAction(TransportCreateIndexAction.TYPE.name()); + + startMigration(DATA_STREAM_FEATURE); + + GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT); + assertBusy(() -> { + GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest) + .get(); + logger.info(Strings.toString(statusResponse)); + assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR)); + }, 30, TimeUnit.SECONDS); + + blockingActionFilter.unblockAllActions(); + ensureGreen(); + + executeMigration(DATA_STREAM_FEATURE); + ensureGreen(); + + assertMetadataAfterMigration(DATA_STREAM_FEATURE); + } + + private void simulateClusterUpgrade() throws Exception { + String indexVersionCreated = systemDataStreamDescriptor.getComposableIndexTemplate() + .template() + .settings() + .get(IndexMetadata.SETTING_VERSION_CREATED); + assertThat(indexVersionCreated, is(NEEDS_UPGRADE_INDEX_VERSION.toString())); + // we can't have NEEDS_UPGRADE_VERSION in settings anymore, + // because those settings will be used in index rollover during data stream migration + // instead we update settings here, kinda simulating upgrade to a new version and restart the cluster + systemDataStreamDescriptor = createSystemDataStreamDescriptor(IndexVersion.current()); + + internalCluster().fullRestart(); + ensureGreen(); + } + + private void createDataStream() throws InterruptedException, ExecutionException { + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + TEST_DATA_STREAM_NAME + ); + AcknowledgedResponse createDSResponse = client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); + assertTrue(createDSResponse.isAcknowledged()); + + ensureGreen(); + } + + public static class DataStreamTestPlugin extends Plugin implements SystemIndexPlugin, ActionPlugin { + @Override + public String getFeatureName() { + return DATA_STREAM_FEATURE; + } + + @Override + public String getFeatureDescription() { + return "Feature to test system data streams migration"; + } + + @Override + public Collection getSystemDataStreamDescriptors() { + return List.of(systemDataStreamDescriptor); + } + } +} diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemIndexMigrationIT.java similarity index 74% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemIndexMigrationIT.java index 018b5e5361711..c5589d705d3e4 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemIndexMigrationIT.java @@ -1,37 +1,25 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.test.InternalTestCluster; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; import static org.hamcrest.Matchers.equalTo; /** @@ -51,14 +39,6 @@ protected boolean forbidPrivateIndexSettings() { return false; } - @Override - protected Collection> nodePlugins() { - List> plugins = new ArrayList<>(super.nodePlugins()); - plugins.add(TestPlugin.class); - plugins.add(ReindexPlugin.class); - return plugins; - } - public void testSystemIndexMigrationCanBeInterruptedWithShutdown() throws Exception { CyclicBarrier taskCreated = new CyclicBarrier(2); CyclicBarrier shutdownCompleted = new CyclicBarrier(2); diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java new file mode 100644 index 0000000000000..f4d7f0a172012 --- /dev/null +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportActionIT.java @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.ingest.common.IngestCommonPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.StartILMRequest; +import org.elasticsearch.xpack.core.ilm.StopILMRequest; +import org.elasticsearch.xpack.core.ilm.action.GetStatusAction; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest; +import org.elasticsearch.xpack.ilm.IndexLifecycle; +import org.elasticsearch.xpack.migrate.MigratePlugin; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +public class CopyLifecycleIndexMetadataTransportActionIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of( + LocalStateCompositeXPackPlugin.class, + MigratePlugin.class, + DataStreamsPlugin.class, + IngestCommonPlugin.class, + IndexLifecycle.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s") + // This just generates less churn and makes it easier to read the log file if needed + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false) + .build(); + } + + public void testCreationDate() { + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + + // so creation date is different + safeSleep(2); + + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); + + // verify source and dest date are actually different before copying + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var indexToSettings = settingsResponse.getIndexToSettings(); + var sourceDate = indexToSettings.get(sourceIndex).getAsLong(IndexMetadata.SETTING_CREATION_DATE, 0L); + { + var destDate = indexToSettings.get(destIndex).getAsLong(IndexMetadata.SETTING_CREATION_DATE, 0L); + assertTrue(sourceDate > 0); + assertTrue(destDate > 0); + assertNotEquals(sourceDate, destDate); + } + + // copy over the metadata + copyMetadata(sourceIndex, destIndex); + + var destDate = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)) + .actionGet() + .getIndexToSettings() + .get(destIndex) + .getAsLong(IndexMetadata.SETTING_CREATION_DATE, 0L); + assertEquals(sourceDate, destDate); + } + + public void testILMState() throws Exception { + + Map phases = Map.of( + "hot", + new Phase( + "hot", + TimeValue.ZERO, + Map.of( + "rollover", + new org.elasticsearch.xpack.core.ilm.RolloverAction(null, null, null, 1L, null, null, null, null, null, null) + ) + ) + ); + + var policyName = "my-policy"; + LifecyclePolicy policy = new LifecyclePolicy(policyName, phases); + PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy); + assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).actionGet()); + + // create data stream with a document and wait for ILM to roll it over + var dataStream = createDataStream(policyName); + createDocument(dataStream); + assertAcked(safeGet(client().execute(ILMActions.START, new StartILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest().indices(dataStream))); + assertTrue(getIndexResponse.indices().length > 1); + }); + // stop ILM so source does not change after copying metadata + assertAcked(safeGet(client().execute(ILMActions.STOP, new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var statusResponse = safeGet( + client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)) + ); + assertEquals(OperationMode.STOPPED, statusResponse.getMode()); + }); + + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest().indices(dataStream))); + for (var backingIndex : getIndexResponse.indices()) { + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); + + IndexMetadata destBefore = getClusterMetadata(destIndex).index(destIndex); + assertNull(destBefore.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY)); + + // copy over the metadata + copyMetadata(backingIndex, destIndex); + + var metadataAfter = getClusterMetadata(backingIndex, destIndex); + IndexMetadata sourceAfter = metadataAfter.index(backingIndex); + IndexMetadata destAfter = metadataAfter.index(destIndex); + assertNotNull(destAfter.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY)); + assertEquals( + sourceAfter.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY), + destAfter.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY) + ); + + } + } + + public void testRolloverInfos() throws Exception { + var dataStream = createDataStream(null); + + // rollover a few times + createDocument(dataStream); + rollover(dataStream); + createDocument(dataStream); + rollover(dataStream); + createDocument(dataStream); + var writeIndex = rollover(dataStream); + + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest().indices(dataStream))); + for (var backingIndex : getIndexResponse.indices()) { + + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); + + var metadataBefore = getClusterMetadata(backingIndex, destIndex); + IndexMetadata source = metadataBefore.index(backingIndex); + IndexMetadata destBefore = metadataBefore.index(destIndex); + + // sanity check not equal before the copy + if (backingIndex.equals(writeIndex)) { + assertTrue(source.getRolloverInfos().isEmpty()); + assertTrue(destBefore.getRolloverInfos().isEmpty()); + } else { + assertNotEquals(source.getRolloverInfos(), destBefore.getRolloverInfos()); + } + + // copy over the metadata + copyMetadata(backingIndex, destIndex); + + // now rollover info should be equal + IndexMetadata destAfter = getClusterMetadata(destIndex).index(destIndex); + assertEquals(source.getRolloverInfos(), destAfter.getRolloverInfos()); + } + } + + private String createDataStream(String ilmPolicy) throws Exception { + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); + + Settings settings = ilmPolicy != null ? Settings.builder().put(IndexMetadata.LIFECYCLE_NAME, ilmPolicy).build() : null; + + String mapping = """ + { + "properties": { + "@timestamp": { + "type":"date" + }, + "data":{ + "type":"keyword" + } + } + } + """; + Template idxTemplate = new Template(settings, new CompressedXContent(mapping), null); + + ComposableIndexTemplate template = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template(idxTemplate) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build(); + + assertAcked( + client().execute( + TransportPutComposableIndexTemplateAction.TYPE, + new TransportPutComposableIndexTemplateAction.Request(dataStreamName + "_template").indexTemplate(template) + ) + ); + assertAcked( + client().execute( + CreateDataStreamAction.INSTANCE, + new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) + ) + ); + return dataStreamName; + } + + private long createDocument(String dataStreamName) throws Exception { + // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order. + long timeSeed = System.currentTimeMillis(); + long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed); + safeGet( + client().index( + new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) + .source( + JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .field("data", randomAlphaOfLength(25)) + .endObject() + ) + ) + ); + safeGet( + indicesAdmin().refresh( + new RefreshRequest(".ds-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden()) + ) + ); + return timestamp; + } + + private void copyMetadata(String sourceIndex, String destIndex) { + assertAcked( + client().execute( + CopyLifecycleIndexMetadataAction.INSTANCE, + new CopyLifecycleIndexMetadataAction.Request(TEST_REQUEST_TIMEOUT, sourceIndex, destIndex) + ) + ); + } + + private String rollover(String dataStream) { + var rolloverResponse = safeGet(indicesAdmin().rolloverIndex(new RolloverRequest(dataStream, null))); + assertTrue(rolloverResponse.isAcknowledged()); + return rolloverResponse.getNewIndex(); + } + + private Metadata getClusterMetadata(String... indices) { + return safeGet(clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT).indices(indices))).getState().metadata(); + } +} diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java index a029f3b9c5ecb..38fbabb43b77b 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java @@ -32,6 +32,8 @@ import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; public class CreateIndexFromSourceActionIT extends ESIntegTestCase { @@ -205,7 +207,7 @@ public void testSettingsNullOverride() throws Exception { assertTrue(destSettings.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)); // override null removed - assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE)); + assertThat(destSettings.keySet(), not(hasItem(IndexMetadata.SETTING_BLOCKS_WRITE))); } public void testRemoveIndexBlocksByDefault() throws Exception { @@ -235,9 +237,9 @@ public void testRemoveIndexBlocksByDefault() throws Exception { var destSettings = settingsResponse.getIndexToSettings().get(destIndex); // remove block settings override both source settings and override settings - assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE)); - assertNull(destSettings.get(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)); - assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_READ)); + assertThat(destSettings.keySet(), not(hasItem(IndexMetadata.SETTING_BLOCKS_WRITE))); + assertThat(destSettings.keySet(), not(hasItem(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE))); + assertThat(destSettings.keySet(), not(hasItem(IndexMetadata.SETTING_BLOCKS_READ))); } public void testMappingsOverridden() { diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java index b6ff76095ac16..156e3da41fc69 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java @@ -69,10 +69,7 @@ public void testAlreadyUpToDateDataStream() throws Exception { dataStreamName ); final int backingIndexCount = createDataStream(dataStreamName); - AcknowledgedResponse response = client().execute( - new ActionType(ReindexDataStreamAction.NAME), - reindexDataStreamRequest - ).actionGet(); + client().execute(new ActionType(ReindexDataStreamAction.NAME), reindexDataStreamRequest).actionGet(); String persistentTaskId = "reindex-data-stream-" + dataStreamName; AtomicReference runningTask = new AtomicReference<>(); for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { @@ -91,12 +88,14 @@ public void testAlreadyUpToDateDataStream() throws Exception { ); } ReindexDataStreamTask task = runningTask.get(); - assertNotNull(task); - assertThat(task.getStatus().complete(), equalTo(true)); - assertNull(task.getStatus().exception()); - assertThat(task.getStatus().pending(), equalTo(0)); - assertThat(task.getStatus().inProgress(), equalTo(Set.of())); - assertThat(task.getStatus().errors().size(), equalTo(0)); + assertBusy(() -> { + assertNotNull(task); + assertThat(task.getStatus().complete(), equalTo(true)); + assertNull(task.getStatus().exception()); + assertThat(task.getStatus().pending(), equalTo(0)); + assertThat(task.getStatus().inProgress(), equalTo(Set.of())); + assertThat(task.getStatus().errors().size(), equalTo(0)); + }); assertBusy(() -> { GetMigrationReindexStatusAction.Response statusResponse = client().execute( diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index bebc71ef3716f..7272d95dc5c14 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -7,8 +7,9 @@ package org.elasticsearch.xpack.migrate.action; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -22,50 +23,109 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.action.ingest.DeletePipelineRequest; +import org.elasticsearch.action.ingest.DeletePipelineTransportAction; +import org.elasticsearch.action.ingest.GetPipelineAction; +import org.elasticsearch.action.ingest.GetPipelineRequest; +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.ingest.PutPipelineTransportAction; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.frozen.FrozenEngine; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.ingest.common.IngestCommonPlugin; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; +import org.elasticsearch.protocol.xpack.frozen.FreezeResponse; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.StartILMRequest; +import org.elasticsearch.xpack.core.ilm.StopILMRequest; +import org.elasticsearch.xpack.core.ilm.action.GetStatusAction; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest; +import org.elasticsearch.xpack.frozen.action.TransportFreezeIndexAction; +import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.migrate.MigratePlugin; +import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry; +import org.junit.Before; import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; +import static java.lang.Boolean.parseBoolean; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { + @Before + private void setup() throws Exception { + safeGet( + clusterAdmin().execute( + DeletePipelineTransportAction.TYPE, + new DeletePipelineRequest(MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME) + ) + ); + + assertBusy(() -> { + assertTrue( + safeGet( + clusterAdmin().execute( + GetPipelineAction.INSTANCE, + new GetPipelineRequest(MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME) + ) + ).isFound() + ); + }); + } + private static final String MAPPING = """ { "_doc":{ "dynamic":"strict", "properties":{ - "foo1":{ - "type":"text" - } + "foo1": {"type":"text"}, + "@timestamp": {"type":"date"} } } } @@ -73,23 +133,153 @@ public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(MigratePlugin.class, ReindexPlugin.class, MockTransportService.TestPlugin.class, DataStreamsPlugin.class); + return List.of( + MigratePlugin.class, + ReindexPlugin.class, + MockTransportService.TestPlugin.class, + DataStreamsPlugin.class, + IngestCommonPlugin.class, + TestFrozenIndicesPlugin.class, + IndexLifecycle.class, + LocalStateCompositeXPackPlugin.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s") + // This just generates less churn and makes it easier to read the log file if needed + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false) + .build(); + } + + private static String DATA_STREAM_MAPPING = """ + { + "dynamic": true, + "_data_stream_timestamp": { + "enabled": true + }, + "properties": { + "@timestamp": {"type":"date"} + } + } + """; + + public void testTimestamp0AddedIfMissing() { + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + + // add doc without timestamp + addDoc(sourceIndex, "{\"foo\":\"baz\"}"); + + // wait until doc is written to all shards before adding mapping + ensureHealth(sourceIndex); + + // add timestamp to source mapping + indicesAdmin().preparePutMapping(sourceIndex).setSource(DATA_STREAM_MAPPING, XContentType.JSON).get(); + + // call reindex + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); + + assertResponse(prepareSearch(destIndex), response -> { + Map sourceAsMap = response.getHits().getAt(0).getSourceAsMap(); + assertEquals(Integer.valueOf(0), sourceAsMap.get(DEFAULT_TIMESTAMP_FIELD)); + }); + } + + public void testTimestampNotAddedIfExists() { + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + + // add doc with timestamp + String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + var doc = String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, time); + addDoc(sourceIndex, doc); + + // wait until doc is written to all shards before adding mapping + ensureHealth(sourceIndex); + + // add timestamp to source mapping + indicesAdmin().preparePutMapping(sourceIndex).setSource(DATA_STREAM_MAPPING, XContentType.JSON).get(); + + // call reindex + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); + + assertResponse(prepareSearch(destIndex), response -> { + Map sourceAsMap = response.getHits().getAt(0).getSourceAsMap(); + assertEquals(time, sourceAsMap.get(DEFAULT_TIMESTAMP_FIELD)); + }); + } + + public void testCustomReindexPipeline() { + String customPipeline = """ + { + "processors": [ + { + "set": { + "field": "cheese", + "value": "gorgonzola" + } + } + ], + "version": 1000 + } + """; + + PutPipelineRequest putRequest = new PutPipelineRequest( + MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME, + new BytesArray(customPipeline), + XContentType.JSON + ); + + safeGet(clusterAdmin().execute(PutPipelineTransportAction.TYPE, putRequest)); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + + // add doc with timestamp + String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + var doc = String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, time); + addDoc(sourceIndex, doc); + + // wait until doc is written to all shards before adding mapping + ensureHealth(sourceIndex); + + // add timestamp to source mapping + indicesAdmin().preparePutMapping(sourceIndex).setSource(DATA_STREAM_MAPPING, XContentType.JSON).get(); + + String destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); + + assertResponse(prepareSearch(destIndex), response -> { + Map sourceAsMap = response.getHits().getAt(0).getSourceAsMap(); + assertEquals("gorgonzola", sourceAsMap.get("cheese")); + assertEquals(time, sourceAsMap.get(DEFAULT_TIMESTAMP_FIELD)); + }); } public void testDestIndexDeletedIfExists() throws Exception { // empty source index var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); // dest index with docs var destIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); - indicesAdmin().create(new CreateIndexRequest(destIndex)).actionGet(); + safeGet(indicesAdmin().create(new CreateIndexRequest(destIndex))); indexDocs(destIndex, 10); - indicesAdmin().refresh(new RefreshRequest(destIndex)).actionGet(); + safeGet(indicesAdmin().refresh(new RefreshRequest(destIndex))); assertHitCount(prepareSearch(destIndex).setSize(0), 10); // call reindex - client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + safeGet(client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))); // verify that dest still exists, but is now empty assertTrue(indexExists(destIndex)); @@ -98,76 +288,67 @@ public void testDestIndexDeletedIfExists() throws Exception { public void testDestIndexNameSet_noDotPrefix() throws Exception { var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); // call reindex - var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet(); + var response = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ); var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); assertEquals(expectedDestIndexName, response.getDestIndex()); } - public void testDestIndexNameSet_withDotPrefix() throws Exception { - + public void testDestIndexNameSet_withDotPrefix() { var sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); // call reindex - var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet(); + var response = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ); var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); assertEquals(expectedDestIndexName, response.getDestIndex()); } - public void testDestIndexContainsDocs() throws Exception { + public void testDestIndexContainsDocs() { // source index with docs var numDocs = randomIntBetween(1, 100); var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); indexDocs(sourceIndex, numDocs); + var settings = Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_METADATA, randomBoolean()) + .put(IndexMetadata.SETTING_READ_ONLY, randomBoolean()) + .build(); + safeGet(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, sourceIndex))); + // call reindex - var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet(); - indicesAdmin().refresh(new RefreshRequest(response.getDestIndex())).actionGet(); + var response = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ); + safeGet(indicesAdmin().refresh(new RefreshRequest(response.getDestIndex()))); // verify that dest contains docs assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs); } - public void testSetSourceToBlockWrites() throws Exception { - var settings = randomBoolean() ? Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build() : Settings.EMPTY; - - // empty source index - var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).get(); - - // call reindex - client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); - - // assert that write to source fails - var indexReq = new IndexRequest(sourceIndex).source(jsonBuilder().startObject().field("field", "1").endObject()); - assertThrows(ClusterBlockException.class, () -> client().index(indexReq).actionGet()); - assertHitCount(prepareSearch(sourceIndex).setSize(0), 0); - } - public void testMissingSourceIndex() { var nonExistentSourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - assertThrows( + expectThrows( ResourceNotFoundException.class, - () -> client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(nonExistentSourceIndex)) - .actionGet() + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(nonExistentSourceIndex)) ); } - public void testSettingsAddedBeforeReindex() throws Exception { + public void testSettingsAddedBeforeReindex() { // start with a static setting var numShards = randomIntBetween(1, 10); var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings)).get(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings))); // update with a dynamic setting var numReplicas = randomIntBetween(0, 10); @@ -176,30 +357,30 @@ public void testSettingsAddedBeforeReindex() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval) .build(); - indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet(); + safeGet(indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex))); // call reindex - var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet() - .getDestIndex(); + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); // assert both static and dynamic settings set on dest index - var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + var settingsResponse = safeGet(indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex))); assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); assertEquals(refreshInterval, settingsResponse.getSetting(destIndex, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey())); } - public void testMappingsAddedToDestIndex() throws Exception { + public void testMappingsAddedToDestIndex() { var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(MAPPING)).actionGet(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(MAPPING))); // call reindex - var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet() - .getDestIndex(); + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = safeGet(indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex))); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -209,34 +390,6 @@ public void testMappingsAddedToDestIndex() throws Exception { assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); } - public void testFailIfMetadataBlockSet() { - var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_METADATA, true).build(); - indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); - - try { - client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); - } catch (ElasticsearchException e) { - assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); - } - - cleanupMetadataBlocks(sourceIndex); - } - - public void testFailIfReadBlockSet() { - var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_READ, true).build(); - indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); - - try { - client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); - } catch (ElasticsearchException e) { - assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); - } - - cleanupMetadataBlocks(sourceIndex); - } - public void testReadOnlyBlocksNotAddedBack() { var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); var settings = Settings.builder() @@ -244,19 +397,18 @@ public void testReadOnlyBlocksNotAddedBack() { .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean()) .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) .build(); - indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings))); // call reindex - var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet() - .getDestIndex(); + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); - var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); - assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); - assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE))); - assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); + var settingsResponse = safeGet(indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex))); + assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); + assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE))); + assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); - cleanupMetadataBlocks(sourceIndex); cleanupMetadataBlocks(destIndex); } @@ -273,11 +425,11 @@ public void testUpdateSettingsDefaultsRestored() { assertAcked(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); // call reindex - var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet() - .getDestIndex(); + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); - var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var settingsResponse = safeGet(indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex))); var destSettings = settingsResponse.getIndexToSettings().get(destIndex); assertEquals( @@ -306,32 +458,32 @@ public void testSettingsAndMappingsFromTemplate() throws IOException { .build(); var request = new TransportPutComposableIndexTemplateAction.Request("logs-template"); request.indexTemplate(template); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + safeGet(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request)); var sourceIndex = "logs-" + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - indicesAdmin().create(new CreateIndexRequest(sourceIndex)).actionGet(); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); { var indexRequest = new IndexRequest(sourceIndex); indexRequest.source("{ \"foo1\": \"cheese\" }", XContentType.JSON); - client().index(indexRequest).actionGet(); + safeGet(client().index(indexRequest)); } // call reindex - var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - .actionGet() - .getDestIndex(); + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); // verify settings from templates copied to dest index { - var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + var settingsResponse = safeGet(indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex))); assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); } // verify mappings from templates copied to dest index { - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = safeGet(indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex))); var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap(); var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap(); assertEquals(sourceMappings, destMappings); @@ -392,7 +544,7 @@ public void testTsdbStartEndSet() throws Exception { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + safeGet(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request)); // index doc Instant time = Instant.now(); @@ -400,13 +552,11 @@ public void testTsdbStartEndSet() throws Exception { { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); indexRequest.source(TSDB_DOC.replace("$time", formatInstant(time)), XContentType.JSON); - var indexResponse = client().index(indexRequest).actionGet(); + var indexResponse = safeGet(client().index(indexRequest)); backingIndexName = indexResponse.getIndex(); } - var sourceSettings = indicesAdmin().getIndex(new GetIndexRequest().indices(backingIndexName)) - .actionGet() - .getSettings() + var sourceSettings = safeGet(indicesAdmin().getIndex(new GetIndexRequest().indices(backingIndexName))).getSettings() .get(backingIndexName); Instant startTime = IndexSettings.TIME_SERIES_START_TIME.get(sourceSettings); Instant endTime = IndexSettings.TIME_SERIES_END_TIME.get(sourceSettings); @@ -418,15 +568,15 @@ public void testTsdbStartEndSet() throws Exception { // force a rollover so can call reindex and delete var rolloverRequest = new RolloverRequest("k8s", null); - var rolloverResponse = indicesAdmin().rolloverIndex(rolloverRequest).actionGet(); + var rolloverResponse = safeGet(indicesAdmin().rolloverIndex(rolloverRequest)); rolloverResponse.getNewIndex(); // call reindex on the original backing index - var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(backingIndexName)) - .actionGet() - .getDestIndex(); + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(backingIndexName)) + ).getDestIndex(); - var destSettings = indicesAdmin().getIndex(new GetIndexRequest().indices(destIndex)).actionGet().getSettings().get(destIndex); + var destSettings = safeGet(indicesAdmin().getIndex(new GetIndexRequest().indices(destIndex))).getSettings().get(destIndex); var destStart = IndexSettings.TIME_SERIES_START_TIME.get(destSettings); var destEnd = IndexSettings.TIME_SERIES_END_TIME.get(destSettings); @@ -437,13 +587,176 @@ public void testTsdbStartEndSet() throws Exception { assertHitCount(prepareSearch(destIndex).setSize(0), 1); } + public void testIndexUnfrozen() { + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + ensureHealth(sourceIndex); + // add doc with timestamp + String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + var doc = String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, time); + addDoc(sourceIndex, doc); + FreezeRequest freezeRequest = new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, sourceIndex); + freezeRequest.setFreeze(true); + FreezeResponse freezeResponse = safeGet(client().execute(FreezeIndexAction.INSTANCE, freezeRequest)); + assertAcked(freezeResponse); + assertThat( + safeGet(admin().indices().getSettings(new GetSettingsRequest().indices(sourceIndex))).getIndexToSettings() + .get(sourceIndex) + .get(FrozenEngine.INDEX_FROZEN.getKey()), + not(equalTo(null)) + ); + + String destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + ).getDestIndex(); + + assertThat( + safeGet(admin().indices().getSettings(new GetSettingsRequest().indices(sourceIndex))).getIndexToSettings() + .get(sourceIndex) + .get(FrozenEngine.INDEX_FROZEN.getKey()), + equalTo(null) + ); + assertThat( + safeGet(admin().indices().getSettings(new GetSettingsRequest().indices(destIndex))).getIndexToSettings() + .get(destIndex) + .get(FrozenEngine.INDEX_FROZEN.getKey()), + equalTo(null) + ); + } + + public void testIndexLifecycleSettingNotCopied() throws Exception { + Map phases = Map.of( + "hot", + new Phase( + "hot", + TimeValue.ZERO, + Map.of( + "rollover", + new org.elasticsearch.xpack.core.ilm.RolloverAction(null, null, null, 1L, null, null, null, null, null, null) + ) + ) + ); + + var policyName = "my-policy"; + LifecyclePolicy policy = new LifecyclePolicy(policyName, phases); + PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy); + assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).actionGet()); + + // create data stream with a document and wait for ILM to roll it over + var dataStream = createDataStream(policyName); + createDocument(dataStream); + + assertAcked(safeGet(client().execute(ILMActions.START, new StartILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest().indices(dataStream))); + assertTrue(getIndexResponse.indices().length >= 2); + }); + stopILM(); + + var dataStreams = safeGet( + indicesAdmin().execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStream }) + ) + ).getDataStreams(); + + assertFalse(dataStreams.isEmpty()); + String writeIndex = dataStreams.get(0).getDataStream().getWriteIndex().getName(); + List indices = dataStreams.get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); + assertTrue(indices.size() >= 2); + + for (var backingIndex : indices) { + if (backingIndex.equals(writeIndex) == false) { + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(backingIndex)) + ).getDestIndex(); + var settingsResponse = safeGet(indicesAdmin().getSettings(new GetSettingsRequest().indices(backingIndex, destIndex))); + assertEquals(policyName, settingsResponse.getSetting(backingIndex, IndexMetadata.LIFECYCLE_NAME)); + assertNull(settingsResponse.getSetting(destIndex, IndexMetadata.LIFECYCLE_NAME)); + } + } + } + + private void stopILM() throws Exception { + assertAcked(safeGet(client().execute(ILMActions.STOP, new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var statusResponse = safeGet( + client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)) + ); + assertEquals(OperationMode.STOPPED, statusResponse.getMode()); + }); + } + + private String createDataStream(String ilmPolicy) throws Exception { + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); + + Settings settings = ilmPolicy != null ? Settings.builder().put(IndexMetadata.LIFECYCLE_NAME, ilmPolicy).build() : null; + + String mapping = """ + { + "properties": { + "@timestamp": { + "type":"date" + }, + "data":{ + "type":"keyword" + } + } + } + """; + Template idxTemplate = new Template(settings, new CompressedXContent(mapping), null); + + ComposableIndexTemplate template = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template(idxTemplate) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build(); + + assertAcked( + client().execute( + TransportPutComposableIndexTemplateAction.TYPE, + new TransportPutComposableIndexTemplateAction.Request(dataStreamName + "_template").indexTemplate(template) + ) + ); + assertAcked( + client().execute( + CreateDataStreamAction.INSTANCE, + new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) + ) + ); + return dataStreamName; + } + + private long createDocument(String dataStreamName) throws Exception { + // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order. + long timeSeed = System.currentTimeMillis(); + long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed); + safeGet( + client().index( + new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) + .source( + JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .field("data", randomAlphaOfLength(25)) + .endObject() + ) + ) + ); + safeGet( + indicesAdmin().refresh( + new RefreshRequest(".ds-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden()) + ) + ); + return timestamp; + } + private static void cleanupMetadataBlocks(String index) { var settings = Settings.builder() .putNull(IndexMetadata.SETTING_READ_ONLY) .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) - .putNull(IndexMetadata.SETTING_BLOCKS_METADATA) - .build(); - assertAcked(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index)).actionGet()); + .putNull(IndexMetadata.SETTING_BLOCKS_METADATA); + updateIndexSettings(settings, index); } private static void indexDocs(String index, int numDocs) { @@ -456,7 +769,7 @@ private static void indexDocs(String index, int numDocs) { .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) ); } - BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + BulkResponse bulkResponse = safeGet(client().bulk(bulkRequest)); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); } @@ -464,12 +777,35 @@ private static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } - private static String getIndexUUID(String index) { - return indicesAdmin().getIndex(new GetIndexRequest().indices(index)) - .actionGet() - .getSettings() - .get(index) - .get(IndexMetadata.SETTING_INDEX_UUID); + void addDoc(String index, String doc) { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE).source(doc, XContentType.JSON)); + safeGet(client().bulk(bulkRequest)); + } + + private void ensureHealth(String index) { + if (cluster().numDataNodes() > 1) { + ensureGreen(index); + } else { + ensureYellow(index); + } } + /* + * This takes the place of the real FrozenIndices plugin. We can't use that one because its EngineFactory conflicts with the one used + * by this test. We only need the settings and the mapping of the FreezeIndexAction though. + */ + public static class TestFrozenIndicesPlugin extends Plugin implements ActionPlugin { + @Override + public List> getSettings() { + return Arrays.asList(FrozenEngine.INDEX_FROZEN); + } + + @Override + public List> getActions() { + List> actions = new ArrayList<>(); + actions.add(new ActionPlugin.ActionHandler<>(FreezeIndexAction.INSTANCE, TransportFreezeIndexAction.class)); + return actions; + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusAction.java similarity index 55% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusAction.java index 23b094fc72b6f..31511d6a752cc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusAction.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionType; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusRequest.java similarity index 65% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusRequest.java index 959197f1e12b6..18287dd280630 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusRequest.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponse.java similarity index 96% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponse.java index f65152a8b72e9..b36eb1bd00f33 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponse.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeAction.java similarity index 54% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeAction.java index d5f881cef9e5c..525c0b78f9be3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeAction.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionType; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeRequest.java similarity index 64% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeRequest.java index 5681043148821..9b8a4000a0e44 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeRequest.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponse.java similarity index 93% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponse.java index 56c0d2eaad900..c6d8adf8a61c0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponse.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusAction.java similarity index 71% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusAction.java index 3a1b311c93ed3..3349d862ffcc5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusAction.java @@ -1,15 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -20,29 +17,29 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.SingleFeatureMigrationResult; +import org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams; +import org.elasticsearch.system_indices.task.SystemIndexMigrationTaskState; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.upgrades.FeatureMigrationResults; -import org.elasticsearch.upgrades.SingleFeatureMigrationResult; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskParams; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskState; import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.stream.Stream; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; +import static org.elasticsearch.indices.SystemIndices.NO_UPGRADE_REQUIRED_INDEX_VERSION; import static org.elasticsearch.indices.SystemIndices.UPGRADED_INDEX_SUFFIX; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; /** * Transport class for the get feature upgrade status action @@ -51,9 +48,6 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA GetFeatureUpgradeStatusRequest, GetFeatureUpgradeStatusResponse> { - public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0; - public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0; - private final SystemIndices systemIndices; @Inject @@ -147,11 +141,11 @@ static List getIndexInfos(ClusterStat (FeatureMigrationResults) state.metadata().custom(FeatureMigrationResults.TYPE) ).map(FeatureMigrationResults::getFeatureStatuses).map(results -> results.get(feature.getName())).orElse(null); - final String failedFeatureName = featureStatus == null ? null : featureStatus.getFailedIndexName(); - final String failedFeatureUpgradedName = failedFeatureName == null ? null : failedFeatureName + UPGRADED_INDEX_SUFFIX; + final String failedResourceName = featureStatus == null ? null : featureStatus.getFailedResourceName(); + final String failedFeatureUpgradedName = failedResourceName == null ? null : failedResourceName + UPGRADED_INDEX_SUFFIX; final Exception exception = featureStatus == null ? null : featureStatus.getException(); - return feature.getIndexDescriptors() + Stream indexInfoStream = feature.getIndexDescriptors() .stream() .flatMap(descriptor -> descriptor.getMatchingIndices(state.metadata()).stream()) .sorted(String::compareTo) @@ -160,11 +154,32 @@ static List getIndexInfos(ClusterStat indexMetadata -> new GetFeatureUpgradeStatusResponse.IndexInfo( indexMetadata.getIndex().getName(), indexMetadata.getCreationVersion(), - (indexMetadata.getIndex().getName().equals(failedFeatureName) + (indexMetadata.getIndex().getName().equals(failedResourceName) || indexMetadata.getIndex().getName().equals(failedFeatureUpgradedName)) ? exception : null ) - ) - .toList(); + ); + + Stream dataStreamsIndexInfoStream = feature.getDataStreamDescriptors() + .stream() + .flatMap(descriptor -> { + Exception dsException = (descriptor.getDataStreamName().equals(failedResourceName)) ? exception : null; + + // we don't know migration of which backing index has failed, + // so, unfortunately, have to report exception for all indices for now + return descriptor.getMatchingIndices(state.metadata()) + .stream() + .sorted(String::compareTo) + .map(index -> state.metadata().index(index)) + .map( + indexMetadata -> new GetFeatureUpgradeStatusResponse.IndexInfo( + indexMetadata.getIndex().getName(), + indexMetadata.getCreationVersion(), + dsException + ) + ); + }); + + return Stream.concat(indexInfoStream, dataStreamsIndexInfoStream).toList(); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportPostFeatureUpgradeAction.java similarity index 85% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportPostFeatureUpgradeAction.java index 57ebe8ef626fd..13cce7c7fc76a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportPostFeatureUpgradeAction.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -24,18 +22,18 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskParams; import java.util.Comparator; import java.util.EnumSet; import java.util.List; import java.util.Set; -import static org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.action.TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; /** * Transport action for post feature upgrade action diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetFeatureUpgradeStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestGetFeatureUpgradeStatusAction.java similarity index 67% rename from server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetFeatureUpgradeStatusAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestGetFeatureUpgradeStatusAction.java index 0a2fc9cc5c92a..47d6ff3b1e10c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetFeatureUpgradeStatusAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestGetFeatureUpgradeStatusAction.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.rest.action.admin.cluster; +package org.elasticsearch.system_indices.rest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusAction; +import org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusRequest; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPostFeatureUpgradeAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestPostFeatureUpgradeAction.java similarity index 67% rename from server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPostFeatureUpgradeAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestPostFeatureUpgradeAction.java index 38e81668dd0eb..97d6e11cc9fa3 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPostFeatureUpgradeAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestPostFeatureUpgradeAction.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.rest.action.admin.cluster; +package org.elasticsearch.system_indices.rest; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.system_indices.action.PostFeatureUpgradeAction; +import org.elasticsearch.system_indices.action.PostFeatureUpgradeRequest; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/FeatureMigrationResults.java similarity index 93% rename from server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/FeatureMigrationResults.java index 1aeab06146834..7fcb29aecebe9 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/FeatureMigrationResults.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; diff --git a/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java similarity index 89% rename from server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java index 45666ba1ca528..cb50cb41a07c7 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -99,7 +97,7 @@ public void onFailure(Exception clusterStateUpdateException) { () -> format( "failed to update cluster state after failed migration of feature [%s] on index [%s]", featureName, - status.getFailedIndexName() + status.getFailedResourceName() ), clusterStateUpdateException ); diff --git a/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SingleFeatureMigrationResult.java similarity index 91% rename from server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SingleFeatureMigrationResult.java index 5dac57fdf8d41..0975b5eea297d 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SingleFeatureMigrationResult.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.SimpleDiffable; @@ -84,7 +82,7 @@ public boolean succeeded() { * Gets the name of the specific index where the migration failure occurred, if the migration failed. */ @Nullable - public String getFailedIndexName() { + public String getFailedResourceName() { return failedIndexName; } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemDataStreamMigrationInfo.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemDataStreamMigrationInfo.java new file mode 100644 index 0000000000000..39e3ea283ee81 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemDataStreamMigrationInfo.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.system_indices.task; + +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemIndices; + +import java.util.stream.Stream; + +final class SystemDataStreamMigrationInfo extends SystemResourceMigrationInfo { + private final DataStream dataStream; + private final String dataStreamName; + + private SystemDataStreamMigrationInfo( + DataStream dataStream, + String dataStreamName, + String featureName, + String origin, + SystemIndices.Feature owningFeature + ) { + super(featureName, origin, owningFeature); + this.dataStreamName = dataStreamName; + this.dataStream = dataStream; + } + + public static SystemDataStreamMigrationInfo build( + DataStream dataStream, + SystemDataStreamDescriptor dataStreamDescriptor, + SystemIndices.Feature feature + ) { + return new SystemDataStreamMigrationInfo( + dataStream, + dataStreamDescriptor.getDataStreamName(), + feature.getName(), + dataStreamDescriptor.getOrigin(), + feature + ); + } + + public String getDataStreamName() { + return dataStreamName; + } + + @Override + protected String getCurrentResourceName() { + return getDataStreamName(); + } + + @Override + Stream getIndices(Metadata metadata) { + return Stream.concat(dataStream.getIndices().stream(), dataStream.getFailureIndices().stream()).map(metadata::getIndexSafe); + } + + @Override + boolean isCurrentIndexClosed() { + // data stream can't be closed + return false; + } +} diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationExecutor.java similarity index 79% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationExecutor.java index 99e739919effc..e15a1d36bdb9f 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationExecutor.java @@ -1,18 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; -import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -24,6 +20,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -31,7 +28,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; /** * Starts the process of migrating system indices. See {@link SystemIndexMigrator} for the actual migration logic. @@ -40,25 +37,22 @@ public class SystemIndexMigrationExecutor extends PersistentTasksExecutor getIndices(Metadata metadata) { + return Stream.of(currentIndex); + } + + /** + * Indicates if the index to be migrated is closed. + */ + @Override + boolean isCurrentIndexClosed() { + return CLOSE.equals(currentIndex.getState()); + } + + /** + * Gets the name to be used for the post-migration index. + */ + String getNextIndexName() { + return currentIndex.getIndex().getName() + SystemIndices.UPGRADED_INDEX_SUFFIX; + } + + /** + * Gets the mappings to be used for the post-migration index. + */ + String getMappings() { + return mapping; + } + + /** + * Gets the settings to be used for the post-migration index. + */ + Settings getSettings() { + return settings; + } + + String getMigrationScript() { + return migrationScript; + } + + /** + * By default, system indices should not be affected by user defined templates, so this + * method should return false in almost all cases. At the moment certain Kibana indices use + * templates, therefore we allow templates to be used on Kibana created system indices until + * Kibana removes the template use on system index creation. + */ + boolean allowsTemplates() { + return allowsTemplates; + } + + @Override + public String toString() { + return "IndexUpgradeInfo[" + + "currentIndex='" + + currentIndex.getIndex().getName() + + "\'" + + ", featureName='" + + featureName + + '\'' + + ", settings=" + + settings + + ", mapping='" + + mapping + + '\'' + + ", origin='" + + origin + + '\''; + } + + static SystemIndexMigrationInfo build( + IndexMetadata currentIndex, + SystemIndexDescriptor descriptor, + SystemIndices.Feature feature, + IndexScopedSettings indexScopedSettings + ) { + final Settings settings; + final String mapping; + if (descriptor.isAutomaticallyManaged()) { + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put(descriptor.getSettings()); + settingsBuilder.remove(IndexMetadata.SETTING_VERSION_CREATED); // Simplifies testing, should never impact real uses. + settings = settingsBuilder.build(); + + mapping = descriptor.getMappings(); + } else { + // Get Settings from old index + settings = copySettingsForNewIndex(currentIndex.getSettings(), indexScopedSettings); + + // Copy mapping from the old index + mapping = currentIndex.mapping().source().string(); + } + return new SystemIndexMigrationInfo( + currentIndex, + feature.getName(), + settings, + mapping, + descriptor.getOrigin(), + descriptor.getMigrationScript(), + feature, + descriptor.allowsTemplates() + ); + } + + private static Settings copySettingsForNewIndex(Settings currentIndexSettings, IndexScopedSettings indexScopedSettings) { + Settings.Builder newIndexSettings = Settings.builder(); + currentIndexSettings.keySet() + .stream() + .filter(settingKey -> indexScopedSettings.isPrivateSetting(settingKey) == false) + .map(indexScopedSettings::get) + .filter(Objects::nonNull) + .filter(setting -> setting.getProperties().contains(Setting.Property.NotCopyableOnResize) == false) + .filter(setting -> setting.getProperties().contains(Setting.Property.PrivateIndex) == false) + .forEach(setting -> { + newIndexSettings.put(setting.getKey(), currentIndexSettings.get(setting.getKey())); + }); + return newIndexSettings.build(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParams.java similarity index 81% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParams.java index acdacac748a63..b6be87b25d762 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParams.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; @@ -19,7 +17,7 @@ import java.io.IOException; -import static org.elasticsearch.upgrades.FeatureMigrationResults.MIGRATION_ADDED_VERSION; +import static org.elasticsearch.system_indices.task.FeatureMigrationResults.MIGRATION_ADDED_VERSION; /** * The params used to initialize {@link SystemIndexMigrator} when it's initially kicked off. diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskState.java similarity index 91% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskState.java index 7bd9a43eca36c..cb7ac812bb8b1 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskState.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; @@ -25,7 +23,7 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; /** diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java similarity index 52% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java index 2215fd33730eb..d5cd346f0402f 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java @@ -1,25 +1,27 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; @@ -27,16 +29,14 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -46,20 +46,30 @@ import org.elasticsearch.persistent.AllocatedPersistentTask; import org.elasticsearch.script.Script; import org.elasticsearch.tasks.TaskId; - -import java.util.LinkedList; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamAction; +import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction; +import org.elasticsearch.xpack.migrate.task.ReindexDataStreamEnrichedStatus; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Queue; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; -import static org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_INDEX_VERSION; +import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; import static org.elasticsearch.cluster.metadata.IndexMetadata.State.CLOSE; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.indices.SystemIndices.NO_UPGRADE_REQUIRED_INDEX_VERSION; /** * This is where the logic to actually perform the migration lives - {@link SystemIndexMigrator#run(SystemIndexMigrationTaskState)} will @@ -74,15 +84,14 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { private final ParentTaskAssigningClient baseClient; private final ClusterService clusterService; private final SystemIndices systemIndices; - private final MetadataUpdateSettingsService metadataUpdateSettingsService; - private final MetadataCreateIndexService metadataCreateIndexService; private final IndexScopedSettings indexScopedSettings; + private final ThreadPool threadPool; // In-memory state // NOTE: This queue is not a thread-safe class. Use `synchronized (migrationQueue)` whenever you access this. I chose this rather than // a synchronized/concurrent collection or an AtomicReference because we often need to do compound operations, which are much simpler // with `synchronized` blocks than when only the collection accesses are protected. - private final Queue migrationQueue = new LinkedList<>(); + private final Queue migrationQueue = new ArrayDeque<>(); private final AtomicReference> currentFeatureCallbackMetadata = new AtomicReference<>(); public SystemIndexMigrator( @@ -91,21 +100,18 @@ public SystemIndexMigrator( String type, String action, TaskId parentTask, - SystemIndexMigrationTaskParams params, Map headers, ClusterService clusterService, SystemIndices systemIndices, - MetadataUpdateSettingsService metadataUpdateSettingsService, - MetadataCreateIndexService metadataCreateIndexService, - IndexScopedSettings indexScopedSettings + IndexScopedSettings indexScopedSettings, + ThreadPool threadPool ) { super(id, type, action, "system-index-migrator", parentTask, headers); this.baseClient = new ParentTaskAssigningClient(client, parentTask); this.clusterService = clusterService; this.systemIndices = systemIndices; - this.metadataUpdateSettingsService = metadataUpdateSettingsService; - this.metadataCreateIndexService = metadataCreateIndexService; this.indexScopedSettings = indexScopedSettings; + this.threadPool = threadPool; } public void run(SystemIndexMigrationTaskState taskState) { @@ -129,7 +135,7 @@ public void run(SystemIndexMigrationTaskState taskState) { return; } - if (stateIndexName != null && clusterState.metadata().hasIndex(stateIndexName) == false) { + if (stateIndexName != null && clusterState.metadata().hasIndexAbstraction(stateIndexName) == false) { markAsFailed(new IndexNotFoundException(stateIndexName, "cannot migrate because that index does not exist")); return; } @@ -147,14 +153,14 @@ public void run(SystemIndexMigrationTaskState taskState) { systemIndices.getFeatures() .stream() - .flatMap(feature -> SystemIndexMigrationInfo.fromFeature(feature, clusterState.metadata(), indexScopedSettings)) - .filter(migrationInfo -> needsToBeMigrated(clusterState.metadata().index(migrationInfo.getCurrentIndexName()))) + .flatMap(feature -> SystemResourceMigrationFactory.fromFeature(feature, clusterState.metadata(), indexScopedSettings)) + .filter(migrationInfo -> needToBeMigrated(migrationInfo.getIndices(clusterState.metadata()))) .sorted() // Stable order between nodes .collect(Collectors.toCollection(() -> migrationQueue)); List closedIndices = migrationQueue.stream() - .filter(SystemIndexMigrationInfo::isCurrentIndexClosed) - .map(SystemIndexMigrationInfo::getCurrentIndexName) + .filter(SystemResourceMigrationInfo::isCurrentIndexClosed) + .map(SystemResourceMigrationInfo::getCurrentResourceName) .toList(); if (closedIndices.isEmpty() == false) { markAsFailed( @@ -166,27 +172,27 @@ public void run(SystemIndexMigrationTaskState taskState) { // The queue we just generated *should* be the same one as was generated on the last node, so the first entry in the queue // should be the same as is in the task state if (stateIndexName != null && stateFeatureName != null && migrationQueue.isEmpty() == false) { - SystemIndexMigrationInfo nextMigrationInfo = migrationQueue.peek(); + SystemResourceMigrationInfo nextMigrationInfo = migrationQueue.peek(); // This should never, ever happen in testing mode, but could conceivably happen if there are different sets of plugins // installed on the previous node vs. this one. assert nextMigrationInfo.getFeatureName().equals(stateFeatureName) - && nextMigrationInfo.getCurrentIndexName().equals(stateIndexName) - : "index name [" + && nextMigrationInfo.getCurrentResourceName().equals(stateIndexName) + : "system index/data stream name [" + stateIndexName + "] or feature name [" + stateFeatureName - + "] from task state did not match first index [" - + nextMigrationInfo.getCurrentIndexName() + + "] from task state did not match first index/data stream [" + + nextMigrationInfo.getCurrentResourceName() + "] and feature [" + nextMigrationInfo.getFeatureName() + "] of locally computed queue, see logs"; - if (nextMigrationInfo.getCurrentIndexName().equals(stateIndexName) == false) { - if (clusterState.metadata().hasIndex(stateIndexName) == false) { + if (nextMigrationInfo.getCurrentResourceName().equals(stateIndexName) == false) { + if (clusterState.metadata().hasIndexAbstraction(stateIndexName) == false) { // If we don't have that index at all, and also don't have the next one markAsFailed( new IllegalStateException( format( - "failed to resume system index migration from index [%s], that index is not present in the cluster", + "failed to resume system index migration from resource [%s], that is not present in the cluster", stateIndexName ) ) @@ -194,8 +200,9 @@ public void run(SystemIndexMigrationTaskState taskState) { } logger.warn( () -> format( - "resuming system index migration with index [%s], which does not match index given in last task state [%s]", - nextMigrationInfo.getCurrentIndexName(), + "resuming system index migration with resource [%s]," + + " which does not match resource given in last task state [%s]", + nextMigrationInfo.getCurrentResourceName(), stateIndexName ) ); @@ -203,70 +210,43 @@ public void run(SystemIndexMigrationTaskState taskState) { } } - // Kick off our callback "loop" - finishIndexAndLoop calls back into prepareNextIndex - cleanUpPreviousMigration( - taskState, - clusterState, - state -> prepareNextIndex(state, state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), stateFeatureName) - ); - } - - private void cleanUpPreviousMigration( - SystemIndexMigrationTaskState taskState, - ClusterState currentState, - Consumer listener - ) { + // Kick off our callback "loop" - finishIndexAndLoop calls back into startFeatureMigration logger.debug("cleaning up previous migration, task state: [{}]", taskState == null ? "null" : Strings.toString(taskState)); - if (taskState != null && taskState.getCurrentIndex() != null) { - SystemIndexMigrationInfo migrationInfo; - try { - migrationInfo = SystemIndexMigrationInfo.fromTaskState( - taskState, - systemIndices, - currentState.metadata(), - indexScopedSettings - ); - } catch (Exception e) { - markAsFailed(e); - return; - } - final String newIndexName = migrationInfo.getNextIndexName(); - logger.info("removing index [{}] from previous incomplete migration", newIndexName); - - migrationInfo.createClient(baseClient) - .admin() - .indices() - .prepareDelete(newIndexName) - .execute(ActionListener.wrap(ackedResponse -> { - if (ackedResponse.isAcknowledged()) { - logger.debug("successfully removed index [{}]", newIndexName); - clearResults(clusterService, ActionListener.wrap(listener::accept, this::markAsFailed)); - } - }, this::markAsFailed)); - } else { - logger.debug("no incomplete index to remove"); - clearResults(clusterService, ActionListener.wrap(listener::accept, this::markAsFailed)); - } + clearResults(clusterService, ActionListener.wrap(state -> startFeatureMigration(stateFeatureName), this::markAsFailed)); } - private void finishIndexAndLoop(BulkByScrollResponse bulkResponse) { + private void finishIndexAndLoop(SystemIndexMigrationInfo migrationInfo, BulkByScrollResponse bulkResponse) { // The BulkByScroll response is validated in #migrateSingleIndex, it's just here to satisfy the ActionListener type assert bulkResponse.isTimedOut() == false && (bulkResponse.getBulkFailures() == null || bulkResponse.getBulkFailures().isEmpty()) && (bulkResponse.getSearchFailures() == null || bulkResponse.getSearchFailures().isEmpty()) : "If this assertion gets triggered it means the validation in migrateSingleIndex isn't working right"; - SystemIndexMigrationInfo lastMigrationInfo = currentMigrationInfo(); logger.info( "finished migrating old index [{}] from feature [{}] to new index [{}]", - lastMigrationInfo.getCurrentIndexName(), - lastMigrationInfo.getFeatureName(), - lastMigrationInfo.getNextIndexName() + migrationInfo.getCurrentIndexName(), + migrationInfo.getFeatureName(), + migrationInfo.getNextIndexName() + ); + + finishResourceAndLoop(migrationInfo); + } + + private void finishDataStreamAndLoop(SystemDataStreamMigrationInfo migrationInfo) { + logger.info( + "finished migrating old indices from data stream [{}] from feature [{}] to new indices", + migrationInfo.getCurrentResourceName(), + migrationInfo.getFeatureName() ); + + finishResourceAndLoop(migrationInfo); + } + + private void finishResourceAndLoop(SystemResourceMigrationInfo lastMigrationInfo) { assert migrationQueue != null && migrationQueue.isEmpty() == false; synchronized (migrationQueue) { migrationQueue.remove(); } - SystemIndexMigrationInfo nextMigrationInfo = currentMigrationInfo(); + SystemResourceMigrationInfo nextMigrationInfo = currentMigrationInfo(); if (nextMigrationInfo == null || nextMigrationInfo.getFeatureName().equals(lastMigrationInfo.getFeatureName()) == false) { // The next feature name is different than the last one, so we just finished a feature - time to invoke its post-migration hook lastMigrationInfo.indicesMigrationComplete( @@ -277,7 +257,8 @@ private void finishIndexAndLoop(BulkByScrollResponse bulkResponse) { if (successful == false) { // GWB> Should we actually fail in this case instead of plugging along? logger.warn( - "post-migration hook for feature [{}] indicated failure; feature migration metadata prior to failure was [{}]", + "post-migration hook for feature [{}] indicated failure;" + + " feature migration metadata prior to failure was [{}]", lastMigrationInfo.getFeatureName(), currentFeatureCallbackMetadata.get() ); @@ -286,30 +267,43 @@ private void finishIndexAndLoop(BulkByScrollResponse bulkResponse) { }, this::markAsFailed) ); } else { - prepareNextIndex( - clusterService.state(), - state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), - lastMigrationInfo.getFeatureName() + startFeatureMigration(lastMigrationInfo.getFeatureName()); + } + } + + private void migrateResource(SystemResourceMigrationInfo migrationInfo, ClusterState clusterState) { + if (migrationInfo instanceof SystemIndexMigrationInfo systemIndexMigrationInfo) { + logger.info( + "preparing to migrate old index [{}] from feature [{}] to new index [{}]", + systemIndexMigrationInfo.getCurrentIndexName(), + migrationInfo.getFeatureName(), + systemIndexMigrationInfo.getNextIndexName() + ); + migrateSingleIndex(systemIndexMigrationInfo, clusterState, this::finishIndexAndLoop); + } else if (migrationInfo instanceof SystemDataStreamMigrationInfo systemDataStreamMigrationInfo) { + logger.info( + "preparing to migrate old indices from data stream [{}] from feature [{}] to new indices", + systemDataStreamMigrationInfo.getCurrentResourceName(), + migrationInfo.getFeatureName() ); + migrateDataStream(systemDataStreamMigrationInfo, this::finishDataStreamAndLoop); + } else { + throw new IllegalStateException("Unknown type of migration: " + migrationInfo.getClass()); } } - private void recordIndexMigrationSuccess(SystemIndexMigrationInfo lastMigrationInfo) { + private void recordIndexMigrationSuccess(SystemResourceMigrationInfo lastMigrationInfo) { MigrationResultsUpdateTask updateTask = MigrationResultsUpdateTask.upsert( lastMigrationInfo.getFeatureName(), SingleFeatureMigrationResult.success(), ActionListener.wrap(state -> { - prepareNextIndex( - state, - clusterState -> migrateSingleIndex(clusterState, this::finishIndexAndLoop), - lastMigrationInfo.getFeatureName() - ); + startFeatureMigration(lastMigrationInfo.getFeatureName()); }, this::markAsFailed) ); updateTask.submit(clusterService); } - private void prepareNextIndex(ClusterState clusterState, Consumer listener, String lastFeatureName) { + private void startFeatureMigration(String lastFeatureName) { synchronized (migrationQueue) { assert migrationQueue != null; if (migrationQueue.isEmpty()) { @@ -319,29 +313,23 @@ private void prepareNextIndex(ClusterState clusterState, Consumer } } - final SystemIndexMigrationInfo migrationInfo = currentMigrationInfo(); + final SystemResourceMigrationInfo migrationInfo = currentMigrationInfo(); assert migrationInfo != null : "the queue of indices to migrate should have been checked for emptiness before calling this method"; - logger.info( - "preparing to migrate old index [{}] from feature [{}] to new index [{}]", - migrationInfo.getCurrentIndexName(), - migrationInfo.getFeatureName(), - migrationInfo.getNextIndexName() - ); if (migrationInfo.getFeatureName().equals(lastFeatureName) == false) { // And then invoke the pre-migration hook for the next one. migrationInfo.prepareForIndicesMigration(clusterService, baseClient, ActionListener.wrap(newMetadata -> { currentFeatureCallbackMetadata.set(newMetadata); - updateTaskState(migrationInfo, listener, newMetadata); + updateTaskState(migrationInfo, state -> migrateResource(migrationInfo, state), newMetadata); }, this::markAsFailed)); } else { // Otherwise, just re-use what we already have. - updateTaskState(migrationInfo, listener, currentFeatureCallbackMetadata.get()); + updateTaskState(migrationInfo, state -> migrateResource(migrationInfo, state), currentFeatureCallbackMetadata.get()); } } - private void updateTaskState(SystemIndexMigrationInfo migrationInfo, Consumer listener, Map metadata) { + private void updateTaskState(SystemResourceMigrationInfo migrationInfo, Consumer listener, Map metadata) { final SystemIndexMigrationTaskState newTaskState = new SystemIndexMigrationTaskState( - migrationInfo.getCurrentIndexName(), + migrationInfo.getCurrentResourceName(), migrationInfo.getFeatureName(), metadata ); @@ -354,16 +342,21 @@ private void updateTaskState(SystemIndexMigrationInfo migrationInfo, Consumer indicesMetadata) { + return indicesMetadata.anyMatch(indexMetadata -> { + assert indexMetadata != null : "null IndexMetadata should be impossible, we're not consistently using the same cluster state"; + if (indexMetadata == null) { + return false; + } + return indexMetadata.isSystem() && indexMetadata.getCreationVersion().before(NO_UPGRADE_REQUIRED_INDEX_VERSION); + }); } - private void migrateSingleIndex(ClusterState clusterState, Consumer listener) { - final SystemIndexMigrationInfo migrationInfo = currentMigrationInfo(); + private void migrateSingleIndex( + SystemIndexMigrationInfo migrationInfo, + ClusterState clusterState, + BiConsumer listener + ) { String oldIndexName = migrationInfo.getCurrentIndexName(); final IndexMetadata imd = clusterState.metadata().index(oldIndexName); if (imd.getState().equals(CLOSE)) { @@ -419,9 +412,12 @@ private void migrateSingleIndex(ClusterState clusterState, Consumer innerListener = ActionListener.wrap(listener::accept, this::markAsFailed); + ActionListener innerListener = ActionListener.wrap( + response -> listener.accept(migrationInfo, response), + this::markAsFailed + ); try { - createIndex(migrationInfo, innerListener.delegateFailureAndWrap((delegate, shardsAcknowledgedResponse) -> { + createIndexRetryOnFailure(migrationInfo, innerListener.delegateFailureAndWrap((delegate, shardsAcknowledgedResponse) -> { logger.debug( "while migrating [{}] , got create index response: [{}]", oldIndexName, @@ -447,12 +443,33 @@ private void migrateSingleIndex(ClusterState clusterState, Consumer { + if (aliasesResponse.hasErrors()) { + var e = new ElasticsearchException("Aliases request had errors"); + for (var error : aliasesResponse.getErrors()) { + e.addSuppressed(error); + } + throw e; + } + logger.info( + "Successfully migrated old index [{}] to new index [{}] from feature [{}]", + oldIndexName, + migrationInfo.getNextIndexName(), + migrationInfo.getFeatureName() + ); + delegate2.onResponse(bulkByScrollResponse); + }, e -> { + logger.error( + () -> format( + "An error occurred while changing aliases and removing the old index [%s] from feature [%s]", + oldIndexName, + migrationInfo.getFeatureName() + ), + e + ); + removeReadOnlyBlockOnReindexFailure(oldIndex, delegate2, e); + })); } }, e -> { logger.error( @@ -484,13 +501,10 @@ private void migrateSingleIndex(ClusterState clusterState, Consumer listener) { - final CreateIndexClusterStateUpdateRequest createRequest = new CreateIndexClusterStateUpdateRequest( - "migrate-system-index", - migrationInfo.getNextIndexName(), - migrationInfo.getNextIndexName() - ); + private void createIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + logger.info("creating new system index [{}] from feature [{}]", migrationInfo.getNextIndexName(), migrationInfo.getFeatureName()); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(migrationInfo.getNextIndexName()); Settings.Builder settingsBuilder = Settings.builder(); if (Objects.nonNull(migrationInfo.getSettings())) { settingsBuilder.put(migrationInfo.getSettings()); @@ -498,16 +512,52 @@ private void createIndex(SystemIndexMigrationInfo migrationInfo, ActionListener< settingsBuilder.remove("index.blocks.read"); settingsBuilder.remove("index.blocks.metadata"); } - createRequest.waitForActiveShards(ActiveShardCount.ALL) - .mappings(migrationInfo.getMappings()) + createIndexRequest.cause(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE) + .ackTimeout(TimeValue.ZERO) + .masterNodeTimeout(TimeValue.MINUS_ONE) + .waitForActiveShards(ActiveShardCount.ALL) + .mapping(migrationInfo.getMappings()) .settings(Objects.requireNonNullElse(settingsBuilder.build(), Settings.EMPTY)); - metadataCreateIndexService.createIndex(TimeValue.MINUS_ONE, TimeValue.ZERO, null, createRequest, listener); + + migrationInfo.createClient(baseClient).admin().indices().create(createIndexRequest, listener); } - private CheckedBiConsumer, AcknowledgedResponse, Exception> setAliasAndRemoveOldIndex( - SystemIndexMigrationInfo migrationInfo, - BulkByScrollResponse bulkByScrollResponse - ) { + private void createIndexRetryOnFailure(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + createIndex(migrationInfo, listener.delegateResponse((l, e) -> { + logger.warn( + "createIndex failed with \"{}\", retrying after removing index [{}] from previous attempt", + e.getMessage(), + migrationInfo.getNextIndexName() + ); + deleteIndex(migrationInfo, ActionListener.wrap(cleanupResponse -> createIndex(migrationInfo, l.delegateResponse((l3, e3) -> { + e3.addSuppressed(e); + logger.error( + "createIndex failed after retrying, aborting system index migration. index: " + migrationInfo.getNextIndexName(), + e3 + ); + l.onFailure(e3); + })), e2 -> { + e2.addSuppressed(e); + logger.error("deleteIndex failed, aborting system index migration. index: " + migrationInfo.getNextIndexName(), e2); + l.onFailure(e2); + })); + })); + } + + private void deleteIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + logger.info("removing index [{}] from feature [{}]", migrationInfo.getNextIndexName(), migrationInfo.getFeatureName()); + String newIndexName = migrationInfo.getNextIndexName(); + migrationInfo.createClient(baseClient).admin().indices().prepareDelete(newIndexName).execute(ActionListener.wrap(ackedResponse -> { + if (ackedResponse.isAcknowledged()) { + logger.info("successfully removed index [{}]", newIndexName); + listener.onResponse(ackedResponse); + } else { + listener.onFailure(new ElasticsearchException("Failed to acknowledge index deletion for [" + newIndexName + "]")); + } + }, listener::onFailure)); + } + + private void setAliasAndRemoveOldIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { final IndicesAliasesRequestBuilder aliasesRequest = migrationInfo.createClient(baseClient).admin().indices().prepareAliases(); aliasesRequest.removeIndex(migrationInfo.getCurrentIndexName()); aliasesRequest.addAlias(migrationInfo.getNextIndexName(), migrationInfo.getCurrentIndexName()); @@ -526,30 +576,35 @@ private CheckedBiConsumer, AcknowledgedResp ); }); - // Technically this callback might have a different cluster state, but it shouldn't matter - these indices shouldn't be changing - // while we're trying to migrate them. - return (listener, unsetReadOnlyResponse) -> aliasesRequest.execute( - listener.delegateFailureAndWrap((l, deleteIndexResponse) -> l.onResponse(bulkByScrollResponse)) - ); + aliasesRequest.execute(listener); } /** - * Makes the index readonly if it's not set as a readonly yet + * Sets the write block on the index to the given value. */ private void setWriteBlock(Index index, boolean readOnlyValue, ActionListener listener) { - final Settings readOnlySettings = Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), readOnlyValue).build(); - - metadataUpdateSettingsService.updateSettings( - new UpdateSettingsClusterStateUpdateRequest( - TimeValue.MINUS_ONE, - TimeValue.ZERO, - readOnlySettings, - UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, - UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REJECT, - index - ), - listener - ); + if (readOnlyValue) { + // Setting the Block with an AddIndexBlockRequest ensures all shards have accounted for the block and all + // in-flight writes are completed before returning. + baseClient.admin() + .indices() + .addBlock( + new AddIndexBlockRequest(WRITE, index.getName()).masterNodeTimeout(TimeValue.MINUS_ONE), + listener.delegateFailureAndWrap((l, response) -> { + if (response.isAcknowledged() == false) { + throw new ElasticsearchException("Failed to acknowledge read-only block index request"); + } + l.onResponse(response); + }) + ); + } else { + // The only way to remove a Block is via a settings update. + final Settings readOnlySettings = Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), false).build(); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(readOnlySettings, index.getName()).setPreserveExisting( + false + ).masterNodeTimeout(TimeValue.MINUS_ONE).ackTimeout(TimeValue.ZERO); + baseClient.execute(TransportUpdateSettingsAction.TYPE, updateSettingsRequest, listener); + } } private void reindex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { @@ -564,12 +619,191 @@ private void reindex(SystemIndexMigrationInfo migrationInfo, ActionListener completionListener + ) { + String dataStreamName = migrationInfo.getDataStreamName(); + logger.info("migrating data stream [{}] from feature [{}]", dataStreamName, migrationInfo.getFeatureName()); + + ReindexDataStreamAction.ReindexDataStreamRequest reindexRequest = new ReindexDataStreamAction.ReindexDataStreamRequest( + ReindexDataStreamAction.Mode.UPGRADE, + dataStreamName + ); + + try { + migrationInfo.createClient(baseClient) + .execute(ReindexDataStreamAction.INSTANCE, reindexRequest, ActionListener.wrap(startMigrationResponse -> { + if (startMigrationResponse.isAcknowledged() == false) { + logger.error("failed to migrate indices from data stream [{}]", dataStreamName); + throw new ElasticsearchException( + "reindex system data stream [" + + dataStreamName + + "] from feature [" + + migrationInfo.getFeatureName() + + "] response is not acknowledge" + ); + } + checkDataStreamMigrationStatus(migrationInfo, completionListener, false); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + // This might happen if the task has been reassigned to another node, + // in this case we can just wait for the data stream migration task to finish. + // But, there is a possibility that previously started data stream migration task has failed, + // in this case we need to cancel it and restart migration of the data stream. + logger.debug("data stream [{}] migration is already in progress", dataStreamName); + checkDataStreamMigrationStatus(migrationInfo, completionListener, true); + } else { + markAsFailed(e); + } + })); + } catch (Exception ex) { + logger.error( + () -> format( + "error occurred while migrating data stream [%s] from feature [%s]", + dataStreamName, + migrationInfo.getFeatureName() + ), + ex + ); + markAsFailed(ex); + } + } + + private void checkDataStreamMigrationStatus( + SystemDataStreamMigrationInfo migrationInfo, + Consumer completionListener, + boolean restartMigrationOnError + ) { + String dataStreamName = migrationInfo.getDataStreamName(); + GetMigrationReindexStatusAction.Request getStatusRequest = new GetMigrationReindexStatusAction.Request(dataStreamName); + + migrationInfo.createClient(baseClient) + .execute(GetMigrationReindexStatusAction.INSTANCE, getStatusRequest, ActionListener.wrap(migrationStatusResponse -> { + ReindexDataStreamEnrichedStatus status = migrationStatusResponse.getEnrichedStatus(); + logger.debug( + "data stream [{}] reindexing status: pending {} out of {} indices", + dataStreamName, + status.pending(), + status.totalIndicesToBeUpgraded() + ); + + if (status.complete() == false) { + // data stream migration task is running, schedule another check without need to cancel-restart + threadPool.schedule( + () -> checkDataStreamMigrationStatus(migrationInfo, completionListener, false), + TimeValue.timeValueSeconds(1), + threadPool.generic() + ); + } else { + List> errors = status.errors(); + if (errors != null && errors.isEmpty() == false || status.exception() != null) { + + // data stream migration task existed before this task started it and is in failed state - cancel it and restart + if (restartMigrationOnError) { + cancelExistingDataStreamMigrationAndRetry(migrationInfo, completionListener); + } else { + List exceptions = (status.exception() != null) + ? Collections.singletonList(status.exception()) + : errors.stream().map(Tuple::v2).toList(); + dataStreamMigrationFailed(migrationInfo, exceptions); + } + } else { + logger.info( + "successfully migrated old indices from data stream [{}] from feature [{}] to new indices", + dataStreamName, + migrationInfo.getFeatureName() + ); + completionListener.accept(migrationInfo); + } + } + }, ex -> cancelExistingDataStreamMigrationAndMarkAsFailed(migrationInfo, ex))); + } + + private void dataStreamMigrationFailed(SystemDataStreamMigrationInfo migrationInfo, Collection exceptions) { + logger.error( + "error occurred while reindexing data stream [{}] from feature [{}], failures [{}]", + migrationInfo.getDataStreamName(), + migrationInfo.getFeatureName(), + exceptions + ); + + ElasticsearchException ex = new ElasticsearchException( + "error occurred while reindexing data stream [" + migrationInfo.getDataStreamName() + "]" + ); + for (Exception exception : exceptions) { + ex.addSuppressed(exception); + } + + throw ex; + } + // Failure handlers private void removeReadOnlyBlockOnReindexFailure(Index index, ActionListener listener, Exception ex) { logger.info("removing read only block on [{}] because reindex failed [{}]", index, ex); setWriteBlock(index, false, ActionListener.wrap(unsetReadOnlyResponse -> listener.onFailure(ex), e1 -> listener.onFailure(ex))); } + private void cancelExistingDataStreamMigrationAndRetry( + SystemDataStreamMigrationInfo migrationInfo, + Consumer completionListener + ) { + logger.debug( + "cancelling migration of data stream [{}] from feature [{}] for retry", + migrationInfo.getDataStreamName(), + migrationInfo.getFeatureName() + ); + + ActionListener listener = ActionListener.wrap(response -> { + if (response.isAcknowledged()) { + migrateDataStream(migrationInfo, completionListener); + } else { + String dataStreamName = migrationInfo.getDataStreamName(); + logger.error( + "failed to cancel migration of data stream [{}] from feature [{}] during retry", + dataStreamName, + migrationInfo.getFeatureName() + ); + throw new ElasticsearchException( + "failed to cancel migration of data stream [" + + dataStreamName + + "] from feature [" + + migrationInfo.getFeatureName() + + "] response is not acknowledge" + ); + } + }, this::markAsFailed); + + cancelDataStreamMigration(migrationInfo, listener); + } + + private void cancelExistingDataStreamMigrationAndMarkAsFailed(SystemDataStreamMigrationInfo migrationInfo, Exception exception) { + logger.info( + "cancelling migration of data stream [{}] from feature [{}]", + migrationInfo.getDataStreamName(), + migrationInfo.getFeatureName() + ); + + // we don't really care here if the request wasn't acknowledged + ActionListener listener = ActionListener.wrap(response -> markAsFailed(exception), ex -> { + exception.addSuppressed(ex); + markAsFailed(exception); + }); + + cancelDataStreamMigration(migrationInfo, listener); + } + + private void cancelDataStreamMigration(SystemDataStreamMigrationInfo migrationInfo, ActionListener listener) { + String dataStreamName = migrationInfo.getDataStreamName(); + + CancelReindexDataStreamAction.Request cancelRequest = new CancelReindexDataStreamAction.Request(dataStreamName); + try { + migrationInfo.createClient(baseClient).execute(CancelReindexDataStreamAction.INSTANCE, cancelRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + private static ElasticsearchException logAndThrowExceptionForFailures(BulkByScrollResponse bulkByScrollResponse) { String bulkFailures = (bulkByScrollResponse.getBulkFailures() != null) ? Strings.collectionToCommaDelimitedString(bulkByScrollResponse.getBulkFailures()) @@ -592,12 +826,16 @@ private static ElasticsearchException logAndThrowExceptionForFailures(BulkByScro */ @Override public void markAsFailed(Exception e) { - SystemIndexMigrationInfo migrationInfo = currentMigrationInfo(); + SystemResourceMigrationInfo migrationInfo = currentMigrationInfo(); synchronized (migrationQueue) { migrationQueue.clear(); } - String featureName = Optional.ofNullable(migrationInfo).map(SystemIndexMigrationInfo::getFeatureName).orElse(""); - String indexName = Optional.ofNullable(migrationInfo).map(SystemIndexMigrationInfo::getCurrentIndexName).orElse(""); + String featureName = Optional.ofNullable(migrationInfo) + .map(SystemResourceMigrationInfo::getFeatureName) + .orElse(""); + String indexName = Optional.ofNullable(migrationInfo) + .map(SystemResourceMigrationInfo::getCurrentResourceName) + .orElse(""); MigrationResultsUpdateTask.upsert( featureName, @@ -647,7 +885,7 @@ private static void submitUnbatchedTask( clusterService.submitUnbatchedStateUpdateTask(source, task); } - private SystemIndexMigrationInfo currentMigrationInfo() { + private SystemResourceMigrationInfo currentMigrationInfo() { synchronized (migrationQueue) { return migrationQueue.peek(); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemResourceMigrationFactory.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemResourceMigrationFactory.java new file mode 100644 index 0000000000000..e7d968673fcea --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemResourceMigrationFactory.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.system_indices.task; + +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.indices.SystemIndices; + +import java.util.Objects; +import java.util.stream.Stream; + +class SystemResourceMigrationFactory { + /** + * Convenience factory method holding the logic for creating instances from a Feature object. + * @param feature The feature that is being migrated + * @param metadata The current metadata, as index migration depends on the current state of the cluster. + * @param indexScopedSettings This is necessary to make adjustments to the indices settings for unmanaged indices. + * @return A {@link Stream} of {@link SystemIndexMigrationInfo}s that represent all the indices the given feature currently owns. + */ + static Stream fromFeature( + SystemIndices.Feature feature, + Metadata metadata, + IndexScopedSettings indexScopedSettings + ) { + return Stream.concat( + getSystemIndicesMigrationInfos(feature, metadata, indexScopedSettings), + getSystemDataStreamsMigrationInfos(feature, metadata) + ); + } + + private static Stream getSystemIndicesMigrationInfos( + SystemIndices.Feature feature, + Metadata metadata, + IndexScopedSettings indexScopedSettings + ) { + return feature.getIndexDescriptors() + .stream() + .flatMap(descriptor -> descriptor.getMatchingIndices(metadata).stream().map(metadata::index).filter(imd -> { + assert imd != null + : "got null IndexMetadata for index in system descriptor [" + + descriptor.getIndexPattern() + + "] in feature [" + + feature.getName() + + "]"; + return Objects.nonNull(imd); + }).map(imd -> SystemIndexMigrationInfo.build(imd, descriptor, feature, indexScopedSettings))); + } + + private static Stream getSystemDataStreamsMigrationInfos( + SystemIndices.Feature feature, + Metadata metadata + ) { + return feature.getDataStreamDescriptors().stream().map(descriptor -> { + DataStream dataStream = metadata.dataStreams().get(descriptor.getDataStreamName()); + return dataStream != null ? SystemDataStreamMigrationInfo.build(dataStream, descriptor, feature) : null; + }).filter(Objects::nonNull); + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemResourceMigrationInfo.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemResourceMigrationInfo.java new file mode 100644 index 0000000000000..e3454dccba737 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemResourceMigrationInfo.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.system_indices.task; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.plugins.SystemIndexPlugin; + +import java.util.Comparator; +import java.util.Map; +import java.util.stream.Stream; + +abstract sealed class SystemResourceMigrationInfo implements Comparable permits SystemDataStreamMigrationInfo, + SystemIndexMigrationInfo { + private static final Comparator SAME_CLASS_COMPARATOR = Comparator.comparing( + SystemResourceMigrationInfo::getFeatureName + ).thenComparing(SystemResourceMigrationInfo::getCurrentResourceName); + + protected final String featureName; + protected final String origin; + protected final SystemIndices.Feature owningFeature; + + SystemResourceMigrationInfo(String featureName, String origin, SystemIndices.Feature owningFeature) { + this.featureName = featureName; + this.origin = origin; + this.owningFeature = owningFeature; + } + + protected abstract String getCurrentResourceName(); + + /** + * Gets the name of the feature which owns the index to be migrated. + */ + String getFeatureName() { + return featureName; + } + + /** + * Gets the origin that should be used when interacting with this index. + */ + String getOrigin() { + return origin; + } + + /** + * Creates a client that's been configured to be able to properly access the system index to be migrated. + * + * @param baseClient The base client to wrap. + * @return An {@link OriginSettingClient} which uses the origin provided by {@link SystemIndexMigrationInfo#getOrigin()}. + */ + Client createClient(Client baseClient) { + return new OriginSettingClient(baseClient, this.getOrigin()); + } + + abstract Stream getIndices(Metadata metadata); + + @Override + public int compareTo(SystemResourceMigrationInfo o) { + return SAME_CLASS_COMPARATOR.compare(this, o); + } + + abstract boolean isCurrentIndexClosed(); + + /** + * Invokes the pre-migration hook for the feature that owns this index. + * See {@link SystemIndexPlugin#prepareForIndicesMigration(ClusterService, Client, ActionListener)}. + * @param clusterService For retrieving the state. + * @param client For performing any update operations necessary to prepare for the upgrade. + * @param listener Call {@link ActionListener#onResponse(Object)} when preparation for migration is complete. + */ + void prepareForIndicesMigration(ClusterService clusterService, Client client, ActionListener> listener) { + owningFeature.getPreMigrationFunction().prepareForIndicesMigration(clusterService, client, listener); + } + + /** + * Invokes the post-migration hooks for the feature that owns this index. + * See {@link SystemIndexPlugin#indicesMigrationComplete(Map, ClusterService, Client, ActionListener)}. + * @param metadata The metadata that was passed into the listener by the pre-migration hook. + * @param clusterService For retrieving the state. + * @param client For performing any update operations necessary to prepare for the upgrade. + * @param listener Call {@link ActionListener#onResponse(Object)} when the hook is finished. + */ + void indicesMigrationComplete( + Map metadata, + ClusterService clusterService, + Client client, + ActionListener listener + ) { + owningFeature.getPostMigrationFunction().indicesMigrationComplete(metadata, clusterService, client, listener); + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java index f5f8beba26d8f..7f134059bb326 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java @@ -7,11 +7,14 @@ package org.elasticsearch.xpack.migrate; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -22,6 +25,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.persistent.PersistentTasksExecutor; @@ -30,12 +34,22 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusAction; +import org.elasticsearch.system_indices.action.PostFeatureUpgradeAction; +import org.elasticsearch.system_indices.action.TransportGetFeatureUpgradeStatusAction; +import org.elasticsearch.system_indices.action.TransportPostFeatureUpgradeAction; +import org.elasticsearch.system_indices.rest.RestGetFeatureUpgradeStatusAction; +import org.elasticsearch.system_indices.rest.RestPostFeatureUpgradeAction; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.SystemIndexMigrationExecutor; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamAction; import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamTransportAction; +import org.elasticsearch.xpack.migrate.action.CopyLifecycleIndexMetadataAction; +import org.elasticsearch.xpack.migrate.action.CopyLifecycleIndexMetadataTransportAction; import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction; import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceTransportAction; import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction; @@ -55,15 +69,33 @@ import org.elasticsearch.xpack.migrate.task.ReindexDataStreamTaskParams; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Stream; import static org.elasticsearch.xpack.core.ClientHelper.REINDEX_DATA_STREAM_ORIGIN; import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexTransportAction.REINDEX_MAX_REQUESTS_PER_SECOND_SETTING; import static org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor.MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING; public class MigratePlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin { + private final SetOnce systemIndices = new SetOnce<>(); + + @Override + public Collection createComponents(PluginServices services) { + systemIndices.set(services.systemIndices()); + + var registry = new MigrateTemplateRegistry( + services.environment().settings(), + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() + ); + registry.initialize(); + return List.of(registry); + } @Override public List getRestHandlers( @@ -82,6 +114,9 @@ public List getRestHandlers( handlers.add(new RestGetMigrationReindexStatusAction()); handlers.add(new RestCancelReindexDataStreamAction()); handlers.add(new RestCreateIndexFromSourceAction()); + + handlers.add(new RestGetFeatureUpgradeStatusAction()); + handlers.add(new RestPostFeatureUpgradeAction()); return handlers; } @@ -93,40 +128,52 @@ public List getRestHandlers( actions.add(new ActionHandler<>(CancelReindexDataStreamAction.INSTANCE, CancelReindexDataStreamTransportAction.class)); actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class)); actions.add(new ActionHandler<>(CreateIndexFromSourceAction.INSTANCE, CreateIndexFromSourceTransportAction.class)); + actions.add(new ActionHandler<>(CopyLifecycleIndexMetadataAction.INSTANCE, CopyLifecycleIndexMetadataTransportAction.class)); + + actions.add(new ActionHandler<>(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class)); + actions.add(new ActionHandler<>(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class)); return actions; } @Override public List getNamedXContent() { - return List.of( - new NamedXContentRegistry.Entry( - PersistentTaskState.class, - new ParseField(ReindexDataStreamPersistentTaskState.NAME), - ReindexDataStreamPersistentTaskState::fromXContent - ), - new NamedXContentRegistry.Entry( - PersistentTaskParams.class, - new ParseField(ReindexDataStreamTaskParams.NAME), - ReindexDataStreamTaskParams::fromXContent + return Stream.concat( + SystemIndexMigrationExecutor.getNamedXContentParsers().stream(), + Stream.of( + new NamedXContentRegistry.Entry( + PersistentTaskState.class, + new ParseField(ReindexDataStreamPersistentTaskState.NAME), + ReindexDataStreamPersistentTaskState::fromXContent + ), + new NamedXContentRegistry.Entry( + PersistentTaskParams.class, + new ParseField(ReindexDataStreamTaskParams.NAME), + ReindexDataStreamTaskParams::fromXContent + ) ) - ); + ).toList(); } @Override public List getNamedWriteables() { - return List.of( - new NamedWriteableRegistry.Entry( - PersistentTaskState.class, - ReindexDataStreamPersistentTaskState.NAME, - ReindexDataStreamPersistentTaskState::new - ), - new NamedWriteableRegistry.Entry( - PersistentTaskParams.class, - ReindexDataStreamTaskParams.NAME, - ReindexDataStreamTaskParams::new - ), - new NamedWriteableRegistry.Entry(Task.Status.class, ReindexDataStreamStatus.NAME, ReindexDataStreamStatus::new) - ); + return Stream.concat( + SystemIndexMigrationExecutor.getNamedWriteables().stream(), + Stream.of( + new NamedWriteableRegistry.Entry(Metadata.Custom.class, FeatureMigrationResults.TYPE, FeatureMigrationResults::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, FeatureMigrationResults.TYPE, FeatureMigrationResults::readDiffFrom), + new NamedWriteableRegistry.Entry( + PersistentTaskState.class, + ReindexDataStreamPersistentTaskState.NAME, + ReindexDataStreamPersistentTaskState::new + ), + new NamedWriteableRegistry.Entry( + PersistentTaskParams.class, + ReindexDataStreamTaskParams.NAME, + ReindexDataStreamTaskParams::new + ), + new NamedWriteableRegistry.Entry(Task.Status.class, ReindexDataStreamStatus.NAME, ReindexDataStreamStatus::new) + ) + ).toList(); } @Override @@ -138,6 +185,13 @@ public List> getPersistentTasksExecutor( IndexNameExpressionResolver expressionResolver ) { return List.of( + new SystemIndexMigrationExecutor( + client, + clusterService, + systemIndices.get(), + settingsModule.getIndexScopedSettings(), + threadPool + ), new ReindexDataStreamPersistentTaskExecutor( new OriginSettingClient(client, REINDEX_DATA_STREAM_ORIGIN), clusterService, diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigrateTemplateRegistry.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigrateTemplateRegistry.java new file mode 100644 index 0000000000000..2a9dc97e16352 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigrateTemplateRegistry.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.migrate; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.IngestPipelineConfig; +import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; + +import java.util.List; + +public class MigrateTemplateRegistry extends IndexTemplateRegistry { + + // This number must be incremented when we make changes to built-in pipeline. + // If a specific user pipeline is needed instead, its version should be set to a value higher than the REGISTRY_VERSION. + static final int REGISTRY_VERSION = 1; + public static final String REINDEX_DATA_STREAM_PIPELINE_NAME = "reindex-data-stream-pipeline"; + private static final String TEMPLATE_VERSION_VARIABLE = "xpack.migrate.reindex.pipeline.version"; + + public MigrateTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + @Override + protected List getIngestPipelines() { + return List.of( + new JsonIngestPipelineConfig( + REINDEX_DATA_STREAM_PIPELINE_NAME, + "/" + REINDEX_DATA_STREAM_PIPELINE_NAME + ".json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ) + ); + } + + @Override + protected String getOrigin() { + return ClientHelper.STACK_ORIGIN; + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java new file mode 100644 index 0000000000000..d2acca1484b0c --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataAction.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class CopyLifecycleIndexMetadataAction extends ActionType { + + public static final String NAME = "indices:admin/index/copy_lifecycle_index_metadata"; + + public static final ActionType INSTANCE = new CopyLifecycleIndexMetadataAction(); + + private CopyLifecycleIndexMetadataAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest implements IndicesRequest { + private final String sourceIndex; + private final String destIndex; + + public Request(TimeValue masterNodeTimeout, String sourceIndex, String destIndex) { + super(masterNodeTimeout, DEFAULT_ACK_TIMEOUT); + this.sourceIndex = sourceIndex; + this.destIndex = destIndex; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.sourceIndex = in.readString(); + this.destIndex = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(sourceIndex); + out.writeString(destIndex); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String sourceIndex() { + return sourceIndex; + } + + public String destIndex() { + return destIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(sourceIndex, request.sourceIndex) && Objects.equals(destIndex, request.destIndex); + } + + @Override + public int hashCode() { + return Objects.hash(sourceIndex, destIndex); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "copying lifecycle metadata for index " + sourceIndex; + } + + @Override + public String[] indices() { + return new String[] { sourceIndex, destIndex }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java new file mode 100644 index 0000000000000..ef263fdda2db3 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CopyLifecycleIndexMetadataTransportAction.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.AckedBatchedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateAckListener; +import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.SimpleBatchedAckListenerTaskExecutor; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.LifecycleExecutionState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.HashMap; + +public class CopyLifecycleIndexMetadataTransportAction extends TransportMasterNodeAction< + CopyLifecycleIndexMetadataAction.Request, + AcknowledgedResponse> { + private static final Logger logger = LogManager.getLogger(CopyLifecycleIndexMetadataTransportAction.class); + private final ClusterStateTaskExecutor executor; + private final MasterServiceTaskQueue taskQueue; + + @Inject + public CopyLifecycleIndexMetadataTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters + ) { + super( + CopyLifecycleIndexMetadataAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + CopyLifecycleIndexMetadataAction.Request::new, + AcknowledgedResponse::readFrom, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.executor = new SimpleBatchedAckListenerTaskExecutor<>() { + @Override + public Tuple executeTask(UpdateIndexMetadataTask task, ClusterState clusterState) { + return new Tuple<>(applyUpdate(clusterState, task), task); + } + }; + this.taskQueue = clusterService.createTaskQueue("migrate-copy-index-metadata", Priority.NORMAL, this.executor); + } + + @Override + protected void masterOperation( + Task task, + CopyLifecycleIndexMetadataAction.Request request, + ClusterState state, + ActionListener listener + ) { + taskQueue.submitTask( + "migrate-copy-index-metadata", + new UpdateIndexMetadataTask(request.sourceIndex(), request.destIndex(), request.ackTimeout(), listener), + request.masterNodeTimeout() + ); + } + + @Override + protected ClusterBlockException checkBlock(CopyLifecycleIndexMetadataAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + private static ClusterState applyUpdate(ClusterState state, UpdateIndexMetadataTask updateTask) { + + IndexMetadata sourceMetadata = state.metadata().index(updateTask.sourceIndex); + if (sourceMetadata == null) { + throw new IndexNotFoundException(updateTask.sourceIndex); + } + IndexMetadata destMetadata = state.metadata().index(updateTask.destIndex); + if (destMetadata == null) { + throw new IndexNotFoundException(updateTask.destIndex); + } + + IndexMetadata.Builder newDestMetadata = IndexMetadata.builder(destMetadata); + + var sourceILM = sourceMetadata.getCustomData(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY); + if (sourceILM != null) { + newDestMetadata.putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, sourceILM); + } + + newDestMetadata.putRolloverInfos(sourceMetadata.getRolloverInfos()) + // creation date is required for ILM to function + .creationDate(sourceMetadata.getCreationDate()) + // creation date updates settings so must increment settings version + .settingsVersion(destMetadata.getSettingsVersion() + 1); + + var indices = new HashMap<>(state.metadata().indices()); + indices.put(updateTask.destIndex, newDestMetadata.build()); + + Metadata newMetadata = Metadata.builder(state.metadata()).indices(indices).build(); + return ClusterState.builder(state).metadata(newMetadata).build(); + } + + static class UpdateIndexMetadataTask extends AckedBatchedClusterStateUpdateTask { + private final String sourceIndex; + private final String destIndex; + + UpdateIndexMetadataTask(String sourceIndex, String destIndex, TimeValue ackTimeout, ActionListener listener) { + super(ackTimeout, listener); + this.sourceIndex = sourceIndex; + this.destIndex = destIndex; + } + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java index 14e5e8cccd910..5ab009decd381 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java @@ -15,6 +15,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -191,5 +194,15 @@ public String[] indices() { public IndicesOptions indicesOptions() { return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "creating index " + destIndex + " from " + sourceIndex; + } } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java index 81edb0e716f51..a12316129a4b5 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java @@ -46,13 +46,13 @@ public class CreateIndexFromSourceTransportAction extends HandledTransportAction private final ClusterService clusterService; private final Client client; private final IndexScopedSettings indexScopedSettings; - private static final Settings REMOVE_INDEX_BLOCKS_SETTING_OVERRIDE = Settings.builder() - .putNull(IndexMetadata.SETTING_READ_ONLY) - .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) - .putNull(IndexMetadata.SETTING_BLOCKS_WRITE) - .putNull(IndexMetadata.SETTING_BLOCKS_METADATA) - .putNull(IndexMetadata.SETTING_BLOCKS_READ) - .build(); + private static final Set INDEX_BLOCK_SETTINGS = Set.of( + IndexMetadata.SETTING_READ_ONLY, + IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, + IndexMetadata.SETTING_BLOCKS_WRITE, + IndexMetadata.SETTING_BLOCKS_METADATA, + IndexMetadata.SETTING_BLOCKS_READ + ); @Inject public CreateIndexFromSourceTransportAction( @@ -89,12 +89,15 @@ protected void doExecute(Task task, CreateIndexFromSourceAction.Request request, Settings.Builder settings = Settings.builder() // first settings from source index - .put(filterSettings(sourceIndex)) - // then override with request settings - .put(request.settingsOverride()); + .put(filterSettings(sourceIndex)); + + if (request.settingsOverride().isEmpty() == false) { + applyOverrides(settings, request.settingsOverride()); + } + if (request.removeIndexBlocks()) { // lastly, override with settings to remove index blocks if requested - settings.put(REMOVE_INDEX_BLOCKS_SETTING_OVERRIDE); + INDEX_BLOCK_SETTINGS.forEach(settings::remove); } Map mergeMappings; @@ -114,6 +117,16 @@ protected void doExecute(Task task, CreateIndexFromSourceAction.Request request, client.admin().indices().create(createIndexRequest, listener.map(response -> response)); } + private void applyOverrides(Settings.Builder settings, Settings overrides) { + overrides.keySet().forEach(key -> { + if (overrides.get(key) != null) { + settings.put(key, overrides.get(key)); + } else { + settings.remove(key); + } + }); + } + private static Map toMap(@Nullable MappingMetadata sourceMapping) { return Optional.ofNullable(sourceMapping) .map(MappingMetadata::source) @@ -122,10 +135,14 @@ private static Map toMap(@Nullable MappingMetadata sourceMapping .orElse(Map.of()); } + @SuppressWarnings("unchecked") private static Map mergeMappings(@Nullable MappingMetadata sourceMapping, Map mappingAddition) throws IOException { Map combinedMappingMap = new HashMap<>(toMap(sourceMapping)); XContentHelper.update(combinedMappingMap, mappingAddition, true); + if (sourceMapping != null && combinedMappingMap.size() == 1 && combinedMappingMap.containsKey(sourceMapping.type())) { + combinedMappingMap = (Map) combinedMappingMap.get(sourceMapping.type()); + } return combinedMappingMap; } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java index 64864491191e5..bfb45875e456c 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/GetMigrationReindexStatusTransportAction.java @@ -145,7 +145,7 @@ private void reportStatus( } /* - * This method feches doc counts for all indices in inProgressIndices (and the indices they are being reindexed into). After + * This method fetches doc counts for all indices in inProgressIndices (and the indices they are being reindexed into). After * successfully fetching those, reportStatus is called. */ private void fetchInProgressStatsAndReportStatus( @@ -173,7 +173,7 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) { if (sourceIndexStats == null) { totalDocsInIndex = 0; } else { - DocsStats totalDocsStats = sourceIndexStats.getTotal().getDocs(); + DocsStats totalDocsStats = sourceIndexStats.getPrimaries().getDocs(); totalDocsInIndex = totalDocsStats == null ? 0 : totalDocsStats.getCount(); } IndexStats migratedIndexStats = indicesStatsResponse.getIndex( @@ -183,7 +183,7 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) { if (migratedIndexStats == null) { reindexedDocsInIndex = 0; } else { - DocsStats reindexedDocsStats = migratedIndexStats.getTotal().getDocs(); + DocsStats reindexedDocsStats = migratedIndexStats.getPrimaries().getDocs(); reindexedDocsInIndex = reindexedDocsStats == null ? 0 : reindexedDocsStats.getCount(); } inProgressMap.put(index, Tuple.tuple(totalDocsInIndex, reindexedDocsInIndex)); diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java index faf8982b79bf0..5ebd2040fbcb1 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java @@ -16,6 +16,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -24,6 +27,7 @@ import java.io.IOException; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.function.Predicate; @@ -144,5 +148,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "reindexing data stream " + sourceDataStream; + } } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java index 2e3fd1b76ed32..dec3cf2901fcc 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java @@ -14,8 +14,12 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; public class ReindexDataStreamIndexAction extends ActionType { @@ -78,6 +82,16 @@ public String[] indices() { public IndicesOptions indicesOptions() { return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public String getDescription() { + return "reindexing data stream index " + sourceIndex; + } } public static class Response extends ActionResponse { diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index 1dd0c6d13002e..e38a93b51f6fa 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; @@ -24,7 +25,9 @@ import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -33,29 +36,43 @@ import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.transport.NoNodeAvailableException; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.frozen.FrozenEngine; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.ScrollableHitSource; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; +import org.elasticsearch.protocol.xpack.frozen.FreezeResponse; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate; +import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; +import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry; +import java.util.Arrays; import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.READ_ONLY; public class ReindexDataStreamIndexTransportAction extends HandledTransportAction< ReindexDataStreamIndexAction.Request, @@ -94,6 +111,14 @@ public class ReindexDataStreamIndexTransportAction extends HandledTransportActio private final ClusterService clusterService; private final Client client; + private final TransportService transportService; + /* + * The following is incremented in order to keep track of the current round-robin position for ingest nodes that we send sliced requests + * to. We bound its random starting value to less than or equal to 2 ^ 30 (the default is Integer.MAX_VALUE or 2 ^ 31 - 1) only so that + * the unit test doesn't fail if it rolls over Integer.MAX_VALUE (since the node selected is the same for Integer.MAX_VALUE and + * Integer.MAX_VALUE + 1). + */ + private final AtomicInteger ingestNodeOffsetGenerator = new AtomicInteger(Randomness.get().nextInt(2 ^ 30)); @Inject public ReindexDataStreamIndexTransportAction( @@ -112,6 +137,7 @@ public ReindexDataStreamIndexTransportAction( ); this.clusterService = clusterService; this.client = client; + this.transportService = transportService; } @Override @@ -131,7 +157,7 @@ protected void doExecute( Settings settingsBefore = sourceIndex.getSettings(); - var hasOldVersion = DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterService.state().metadata(), false); + var hasOldVersion = DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterService.state().metadata(), false, true); if (hasOldVersion.test(sourceIndex.getIndex()) == false) { logger.warn( "Migrating index [{}] with version [{}] is unnecessary as its version is not before [{}]", @@ -141,26 +167,21 @@ protected void doExecute( ); } - if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)) { - var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a read block.", destIndexName); - listener.onFailure(new ElasticsearchException(errorMessage)); - return; - } - if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_METADATA, false)) { - var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a metadata block.", destIndexName); - listener.onFailure(new ElasticsearchException(errorMessage)); - return; - } final boolean wasClosed = isClosed(sourceIndex); - SubscribableListener.newForked(l -> setBlockWrites(sourceIndexName, l, taskId)) + + SubscribableListener.newForked(l -> removeMetadataBlocks(sourceIndexName, taskId, l)) + .andThen(l -> unfreezeIfFrozen(sourceIndexName, sourceIndex, l, taskId)) .andThen(l -> openIndexIfClosed(sourceIndexName, wasClosed, l, taskId)) + .andThen(l -> setReadOnly(sourceIndexName, l, taskId)) .andThen(l -> refresh(sourceIndexName, l, taskId)) .andThen(l -> deleteDestIfExists(destIndexName, l, taskId)) .andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId)) .andThen(l -> reindex(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> copyOldSourceSettingsToDest(settingsBefore, destIndexName, l, taskId)) + .andThen(l -> copyIndexMetadataToDest(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> sanityCheck(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> closeIndexIfWasClosed(destIndexName, wasClosed, l, taskId)) + .andThen(l -> removeAPIBlocks(sourceIndexName, taskId, l, READ_ONLY)) .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) .addListener(listener); } @@ -196,9 +217,25 @@ private static boolean isClosed(IndexMetadata indexMetadata) { return indexMetadata.getState().equals(IndexMetadata.State.CLOSE); } - private void setBlockWrites(String sourceIndexName, ActionListener listener, TaskId parentTaskId) { - logger.debug("Setting write block on source index [{}]", sourceIndexName); - addBlockToIndex(WRITE, sourceIndexName, new ActionListener<>() { + private void unfreezeIfFrozen( + String sourceIndexName, + IndexMetadata indexMetadata, + ActionListener listener, + TaskId parentTaskId + ) { + if (FrozenEngine.INDEX_FROZEN.get(indexMetadata.getSettings()).equals(Boolean.TRUE)) { + logger.debug("Unfreezing source index [{}]", sourceIndexName); + FreezeRequest freezeRequest = new FreezeRequest(TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, sourceIndexName).setFreeze(false); + freezeRequest.setParentTask(parentTaskId); + client.execute(FreezeIndexAction.INSTANCE, freezeRequest, listener); + } else { + listener.onResponse(null); + } + } + + private void setReadOnly(String sourceIndexName, ActionListener listener, TaskId parentTaskId) { + logger.debug("Setting read-only on source index [{}]", sourceIndexName); + addBlockToIndex(READ_ONLY, sourceIndexName, new ActionListener<>() { @Override public void onResponse(AddIndexBlockResponse response) { if (response.isAcknowledged()) { @@ -249,6 +286,8 @@ private void createIndex( var settingsOverride = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) + // remove lifecycle so that ILM does not start processing before the index is added to data stream + .putNull(IndexMetadata.LIFECYCLE_NAME) .build(); var request = new CreateIndexFromSourceAction.Request( @@ -268,13 +307,62 @@ void reindex(String sourceIndexName, String destIndexName, ActionListener checkForFailuresListener = ActionListener.wrap(bulkByScrollResponse -> { + if (bulkByScrollResponse.getSearchFailures().isEmpty() == false) { + ScrollableHitSource.SearchFailure firstSearchFailure = bulkByScrollResponse.getSearchFailures().get(0); + listener.onFailure( + new ElasticsearchException( + "Failure reading data from {} caused by {}", + firstSearchFailure.getReason(), + sourceIndexName, + firstSearchFailure.getReason().getMessage() + ) + ); + } else if (bulkByScrollResponse.getBulkFailures().isEmpty() == false) { + BulkItemResponse.Failure firstBulkFailure = bulkByScrollResponse.getBulkFailures().get(0); + listener.onFailure( + new ElasticsearchException( + "Failure loading data from {} into {} caused by {}", + firstBulkFailure.getCause(), + sourceIndexName, + destIndexName, + firstBulkFailure.getCause().getMessage() + ) + ); + } else { + listener.onResponse(bulkByScrollResponse); + } + }, listener::onFailure); + /* + * Reindex will potentially run a pipeline for each document. If we run all reindex requests on the same node (locally), that + * becomes a bottleneck. This code round-robins reindex requests to all ingest nodes to spread out the pipeline workload. When a + * data stream has many indices, this can improve performance a good bit. + */ + final DiscoveryNode[] ingestNodes = clusterService.state().getNodes().getIngestNodes().values().toArray(DiscoveryNode[]::new); + if (ingestNodes.length == 0) { + listener.onFailure(new NoNodeAvailableException("No ingest nodes in cluster")); + } else { + DiscoveryNode ingestNode = ingestNodes[Math.floorMod(ingestNodeOffsetGenerator.incrementAndGet(), ingestNodes.length)]; + logger.debug("Sending reindex request to {}", ingestNode.getName()); + transportService.sendRequest( + ingestNode, + ReindexAction.NAME, + reindexRequest, + new ActionListenerResponseHandler<>( + checkForFailuresListener, + BulkByScrollResponse::new, + TransportResponseHandler.TRANSPORT_WORKER + ) + ); + } } private void updateSettings( @@ -302,6 +390,24 @@ private void copyOldSourceSettingsToDest( updateSettings(destIndexName, settings, listener, parentTaskId); } + private void copyIndexMetadataToDest( + String sourceIndexName, + String destIndexName, + ActionListener listener, + TaskId parentTaskId + ) { + logger.debug("Copying index metadata to destination index [{}] from source index [{}]", destIndexName, sourceIndexName); + var request = new CopyLifecycleIndexMetadataAction.Request(TimeValue.MAX_VALUE, sourceIndexName, destIndexName); + request.setParentTask(parentTaskId); + var errorMessage = String.format( + Locale.ROOT, + "Failed to acknowledge copying index metadata from source [%s] to dest [%s]", + sourceIndexName, + destIndexName + ); + client.execute(CopyLifecycleIndexMetadataAction.INSTANCE, request, failIfNotAcknowledged(listener, errorMessage)); + } + private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder builder, String setting) { // if setting was explicitly added to the source index if (settingsBefore.get(setting) != null) { @@ -341,10 +447,34 @@ private void addBlockToIndex( TaskId parentTaskId ) { AddIndexBlockRequest addIndexBlockRequest = new AddIndexBlockRequest(block, index); + addIndexBlockRequest.markVerified(false); addIndexBlockRequest.setParentTask(parentTaskId); client.admin().indices().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest, listener); } + /** + * All metadata blocks need to be removed at the start for the following reasons: + * 1) If the source index has a metadata only block, the read-only block can't be added. + * 2) If the source index is read-only and closed, it can't be opened. + */ + private void removeMetadataBlocks(String indexName, TaskId parentTaskId, ActionListener listener) { + logger.debug("Removing metadata blocks from index [{}]", indexName); + removeAPIBlocks(indexName, parentTaskId, listener, METADATA, READ_ONLY); + } + + private void removeAPIBlocks( + String indexName, + TaskId parentTaskId, + ActionListener listener, + IndexMetadata.APIBlock... blocks + ) { + Settings.Builder settings = Settings.builder(); + Arrays.stream(blocks).forEach(b -> settings.putNull(b.settingName())); + var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), indexName); + updateSettingsRequest.setParentTask(parentTaskId); + client.execute(TransportUpdateSettingsAction.TYPE, updateSettingsRequest, listener); + } + private void getIndexDocCount(String index, TaskId parentTaskId, ActionListener listener) { SearchRequest countRequest = new SearchRequest(index); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true); @@ -366,26 +496,24 @@ private void sanityCheck( ) { if (Assertions.ENABLED) { logger.debug("Comparing source [{}] and dest [{}] doc counts", sourceIndexName, destIndexName); - client.execute( - RefreshAction.INSTANCE, - new RefreshRequest(destIndexName), - listener.delegateFailureAndWrap((delegate, ignored) -> { - getIndexDocCount(sourceIndexName, parentTaskId, delegate.delegateFailureAndWrap((delegate1, sourceCount) -> { - getIndexDocCount(destIndexName, parentTaskId, delegate1.delegateFailureAndWrap((delegate2, destCount) -> { - assert sourceCount == destCount - : String.format( - Locale.ROOT, - "source index [%s] has %d docs and dest [%s] has %d docs", - sourceIndexName, - sourceCount, - destIndexName, - destCount - ); - delegate2.onResponse(null); - })); + RefreshRequest refreshRequest = new RefreshRequest(destIndexName); + refreshRequest.setParentTask(parentTaskId); + client.execute(RefreshAction.INSTANCE, refreshRequest, listener.delegateFailureAndWrap((delegate, ignored) -> { + getIndexDocCount(sourceIndexName, parentTaskId, delegate.delegateFailureAndWrap((delegate1, sourceCount) -> { + getIndexDocCount(destIndexName, parentTaskId, delegate1.delegateFailureAndWrap((delegate2, destCount) -> { + assert Objects.equals(sourceCount, destCount) + : String.format( + Locale.ROOT, + "source index [%s] has %d docs and dest [%s] has %d docs", + sourceIndexName, + sourceCount, + destIndexName, + destCount + ); + delegate2.onResponse(null); })); - }) - ); + })); + })); } else { listener.onResponse(null); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java index 74545153fd788..e13c4a660a3d3 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java @@ -13,10 +13,12 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -37,13 +39,15 @@ public class ReindexDataStreamTransportAction extends HandledTransportAction { + if (getResponse.getEnrichedStatus().complete() == false) { + throw new ResourceAlreadyExistsException("Reindex task for data stream [{}] already exists", sourceDataStreamName); + } + CancelReindexDataStreamAction.Request cancelRequest = new CancelReindexDataStreamAction.Request(sourceDataStreamName); + cancelRequest.setParentTask(task.getParentTaskId()); + client.execute( + CancelReindexDataStreamAction.INSTANCE, + cancelRequest, + getListener.delegateFailureAndWrap( + (cancelListener, cancelResponse) -> startTask(cancelListener, persistentTaskId, params) + ) + ); + }) + ); + } + + } + + private void startTask(ActionListener listener, String persistentTaskId, ReindexDataStreamTaskParams params) { persistentTasksService.sendStartRequest( persistentTaskId, ReindexDataStreamTask.TASK_NAME, diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java index d9645cc817d2e..226446e8101c1 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java @@ -15,6 +15,10 @@ import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.support.CountDownActionListener; @@ -23,8 +27,10 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -34,6 +40,8 @@ import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.RetryActionRequest; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction; import java.util.ArrayList; @@ -87,7 +95,7 @@ protected ReindexDataStreamTask createTask( id, type, action, - "id=" + taskInProgress.getId(), + "Reindexing data stream " + taskInProgress.getParams().getSourceDataStream(), parentTaskId, headers ); @@ -110,7 +118,8 @@ protected void nodeOperation( List dataStreamInfos = response.getDataStreams(); if (dataStreamInfos.size() == 1) { DataStream dataStream = dataStreamInfos.get(0).getDataStream(); - if (getReindexRequiredPredicate(clusterService.state().metadata(), false).test(dataStream.getWriteIndex())) { + boolean includeSystem = dataStream.isSystem(); + if (getReindexRequiredPredicate(clusterService.state().metadata(), false, includeSystem).test(dataStream.getWriteIndex())) { RolloverRequest rolloverRequest = new RolloverRequest(sourceDataStream, null); rolloverRequest.setParentTask(taskId); client.execute( @@ -157,7 +166,7 @@ private void reindexIndices( ) { List indices = dataStream.getIndices(); List indicesToBeReindexed = indices.stream() - .filter(getReindexRequiredPredicate(clusterService.state().metadata(), false)) + .filter(getReindexRequiredPredicate(clusterService.state().metadata(), false, dataStream.isSystem())) .toList(); final ReindexDataStreamPersistentTaskState updatedState; if (params.totalIndices() != totalIndicesInDataStream @@ -214,9 +223,8 @@ private void maybeProcessNextIndex( SubscribableListener.newForked( l -> client.execute(ReindexDataStreamIndexAction.INSTANCE, reindexDataStreamIndexRequest, l) ) - .andThen( - (l, result) -> updateDataStream(sourceDataStream, index.getName(), result.getDestIndex(), l, parentTaskId) - ) + .andThen((l, result) -> updateDataStream(sourceDataStream, index.getName(), result.getDestIndex(), l, parentTaskId)) + .andThen((l, newIndex) -> copySettings(index.getName(), newIndex, l, parentTaskId)) .andThen(l -> deleteIndex(index.getName(), parentTaskId, l)) .addListener(ActionListener.wrap(unused -> { reindexDataStreamTask.reindexSucceeded(index.getName()); @@ -225,6 +233,7 @@ private void maybeProcessNextIndex( }, e -> { reindexDataStreamTask.reindexFailed(index.getName(), e); listener.onResponse(null); + maybeProcessNextIndex(indicesRemaining, reindexDataStreamTask, sourceDataStream, listener, parentTaskId); })); } @@ -232,7 +241,7 @@ private void updateDataStream( String dataStream, String oldIndex, String newIndex, - ActionListener listener, + ActionListener listener, TaskId parentTaskId ) { ModifyDataStreamsAction.Request modifyDataStreamRequest = new ModifyDataStreamsAction.Request( @@ -241,7 +250,49 @@ private void updateDataStream( List.of(DataStreamAction.removeBackingIndex(dataStream, oldIndex), DataStreamAction.addBackingIndex(dataStream, newIndex)) ); modifyDataStreamRequest.setParentTask(parentTaskId); - client.execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest, listener); + client.execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest, listener.map(ingored -> newIndex)); + } + + /** + * Copy lifecycle name from the old index to the new index, so that ILM can now process the new index. + * If the new index has a lifecycle name before it is swapped into the data stream, ILM will try, and fail, to process + * the new index. For this reason, lifecycle is not set until after the new index has been added to the data stream. + */ + private void copySettings(String oldIndex, String newIndex, ActionListener listener, TaskId parentTaskId) { + var getSettingsRequest = new GetSettingsRequest().indices(oldIndex); + getSettingsRequest.setParentTask(parentTaskId); + client.execute(GetSettingsAction.INSTANCE, getSettingsRequest, listener.delegateFailure((delegate, response) -> { + String lifecycleName = response.getSetting(oldIndex, IndexMetadata.LIFECYCLE_NAME); + if (lifecycleName != null) { + var settings = Settings.builder().put(IndexMetadata.LIFECYCLE_NAME, lifecycleName).build(); + var updateSettingsRequest = new UpdateSettingsRequest(settings, newIndex); + updateSettingsRequest.setParentTask(parentTaskId); + client.execute( + TransportUpdateSettingsAction.TYPE, + updateSettingsRequest, + delegate.delegateFailure((delegate2, response2) -> { + maybeRunILMAsyncAction(newIndex, delegate2, parentTaskId); + }) + ); + } else { + delegate.onResponse(null); + } + })); + } + + /** + * If ILM runs an async action on the source index shortly before reindexing, the results of the async action + * may not yet be in the source index. For example, if a force merge has just been started by ILM, the reindex + * will see the un-force-merged index. But the ILM state will be copied to destination index saying that an + * async action was started, and so ILM won't force merge the destination index. To be sure that the async + * action is run on the destination index, we force a retry on async actions after adding the ILM policy + * to the destination index. + */ + private void maybeRunILMAsyncAction(String newIndex, ActionListener listener, TaskId parentTaskId) { + var retryActionRequest = new RetryActionRequest(TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, newIndex); + retryActionRequest.setParentTask(parentTaskId); + retryActionRequest.requireError(false); + client.execute(ILMActions.RETRY, retryActionRequest, listener); } private void deleteIndex(String indexName, TaskId parentTaskId, ActionListener listener) { diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java index 04295a7521479..d446593e82ac0 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java @@ -55,6 +55,7 @@ public ReindexDataStreamTask( this.persistentTaskStartTime = persistentTaskStartTime; this.initialTotalIndices = initialTotalIndices; this.initialTotalIndicesToBeUpgraded = initialTotalIndicesToBeUpgraded; + this.pending.set(initialTotalIndicesToBeUpgraded); this.completeTask = new RunOnce(() -> { if (exception == null) { markAsCompleted(); diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy b/x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy similarity index 76% rename from qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy rename to x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy index da4792e587d05..db02e9267218a 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy +++ b/x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy @@ -8,7 +8,6 @@ */ grant { - // needed to cause problems - permission java.lang.RuntimePermission "getClassLoader"; - permission java.lang.RuntimePermission "setFactory"; + // needed for Painless to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; }; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponseTests.java similarity index 76% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponseTests.java index ea538e5b085cc..781cf277a26dd 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponseTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.IndexVersion; @@ -16,10 +14,10 @@ import java.util.Collections; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -37,7 +35,7 @@ protected Writeable.Reader instanceReader() { protected GetFeatureUpgradeStatusResponse createTestInstance() { return new GetFeatureUpgradeStatusResponse( randomList(8, GetFeatureUpgradeStatusResponseTests::createFeatureStatus), - randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) + randomFrom(GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) ); } @@ -51,10 +49,7 @@ protected GetFeatureUpgradeStatusResponse mutateInstance(GetFeatureUpgradeStatus GetFeatureUpgradeStatusResponseTests::createFeatureStatus ) ), - randomValueOtherThan( - instance.getUpgradeStatus(), - () -> randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) - ) + randomValueOtherThan(instance.getUpgradeStatus(), () -> randomFrom(GetFeatureUpgradeStatusResponse.UpgradeStatus.values())) ); } @@ -92,7 +87,7 @@ private static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus createFeatur return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus( randomAlphaOfLengthBetween(3, 20), randomFrom(IndexVersion.current(), IndexVersions.MINIMUM_COMPATIBLE), - randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()), + randomFrom(GetFeatureUpgradeStatusResponse.UpgradeStatus.values()), randomList(4, GetFeatureUpgradeStatusResponseTests::getIndexInfo) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponseTests.java similarity index 85% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponseTests.java index 13b6a268ab21f..33ee910b70179 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponseTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.Writeable; diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusActionTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusActionTests.java new file mode 100644 index 0000000000000..489b1bbef4f61 --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusActionTests.java @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.system_indices.action; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemDataStreamDescriptor; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndexDescriptorUtils; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.SingleFeatureMigrationResult; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase { + + private static final String DATA_STREAM_NAME = ".test-ds"; + private static final String BACKING_INDEX_NAME = DataStream.BACKING_INDEX_PREFIX + DATA_STREAM_NAME + "-1"; + private static final String FEATURE_NAME = "test-feature"; + private static String TEST_SYSTEM_INDEX_PATTERN = ".test*"; + private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(6000099); + private static final ClusterState CLUSTER_STATE = getClusterState(); + private static final String TEST_INDEX_1_NAME = ".test-index-1"; + + private static final SystemIndices.Feature FEATURE = getFeature(); + + public void testGetFeatureStatus() { + GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus status = TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus( + CLUSTER_STATE, + FEATURE + ); + + assertThat(status.getUpgradeStatus(), equalTo(MIGRATION_NEEDED)); + assertThat(status.getFeatureName(), equalTo(FEATURE_NAME)); + assertThat(status.getMinimumIndexVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(status.getIndexVersions(), hasSize(3)); // additional testing below + } + + public void testGetIndexInfos() { + List versions = TransportGetFeatureUpgradeStatusAction.getIndexInfos( + CLUSTER_STATE, + FEATURE + ); + + assertThat(versions, hasSize(3)); + + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(0); + assertThat(version.getVersion(), equalTo(IndexVersion.current())); + assertThat(version.getIndexName(), equalTo(TEST_INDEX_1_NAME)); + } + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(1); + assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(version.getIndexName(), equalTo(".test-index-2")); + } + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(2); + assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(version.getIndexName(), equalTo(BACKING_INDEX_NAME)); + } + } + + public void testGetIndexInfosWithErrors() { + List versions = TransportGetFeatureUpgradeStatusAction.getIndexInfos( + getClusterStateWithFailedMigration(TEST_INDEX_1_NAME), + FEATURE + ); + + assertThat(versions, hasSize(3)); + + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(0); + assertThat(version.getVersion(), equalTo(IndexVersion.current())); + assertThat(version.getIndexName(), equalTo(TEST_INDEX_1_NAME)); + assertNotNull(version.getException()); + } + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(1); + assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(version.getIndexName(), equalTo(".test-index-2")); + assertNull(version.getException()); + } + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(2); + assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(version.getIndexName(), equalTo(BACKING_INDEX_NAME)); + assertNull(version.getException()); + } + } + + public void testGetIndexInfosWithDataStreamErrors() { + List versions = TransportGetFeatureUpgradeStatusAction.getIndexInfos( + getClusterStateWithFailedMigration(DATA_STREAM_NAME), + FEATURE + ); + + assertThat(versions, hasSize(3)); + + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(0); + assertThat(version.getVersion(), equalTo(IndexVersion.current())); + assertThat(version.getIndexName(), equalTo(TEST_INDEX_1_NAME)); + assertNull(version.getException()); + } + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(1); + assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(version.getIndexName(), equalTo(".test-index-2")); + assertNull(version.getException()); + } + { + GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(2); + assertThat(version.getVersion(), equalTo(TEST_OLD_VERSION)); + assertThat(version.getIndexName(), equalTo(BACKING_INDEX_NAME)); + assertNotNull(version.getException()); + } + } + + private static SystemIndices.Feature getFeature() { + SystemIndexDescriptor descriptor = SystemIndexDescriptorUtils.createUnmanaged(TEST_SYSTEM_INDEX_PATTERN, "descriptor for tests"); + SystemDataStreamDescriptor dataStreamDescriptor = new SystemDataStreamDescriptor( + DATA_STREAM_NAME, + "test data stream", + SystemDataStreamDescriptor.Type.INTERNAL, + ComposableIndexTemplate.builder().build(), + Map.of(), + Collections.singletonList("origin"), + "origin", + ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ); + + // system indices feature object + SystemIndices.Feature feature = new SystemIndices.Feature( + FEATURE_NAME, + "feature for tests", + List.of(descriptor), + List.of(dataStreamDescriptor) + ); + return feature; + } + + private static ClusterState getClusterState() { + IndexMetadata indexMetadata1 = IndexMetadata.builder(TEST_INDEX_1_NAME) + .settings(Settings.builder().put("index.version.created", IndexVersion.current()).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .system(true) + .build(); + + @UpdateForV9 // Once we start testing 9.x, we should update this test to use a 7.x "version created" + IndexMetadata indexMetadata2 = IndexMetadata.builder(".test-index-2") + .settings(Settings.builder().put("index.version.created", TEST_OLD_VERSION).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .system(true) + .build(); + IndexMetadata dsIndexMetadata = IndexMetadata.builder(BACKING_INDEX_NAME) + .settings(Settings.builder().put("index.version.created", TEST_OLD_VERSION).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .system(true) + .build(); + + DataStream dataStream = DataStream.builder(DATA_STREAM_NAME, List.of(dsIndexMetadata.getIndex())) + .setSystem(true) + .setHidden(true) + .build(); + + ClusterState clusterState = new ClusterState.Builder(ClusterState.EMPTY_STATE).metadata( + new Metadata.Builder().dataStreams(Map.of(DATA_STREAM_NAME, dataStream), Collections.emptyMap()) + .indices(Map.of(TEST_INDEX_1_NAME, indexMetadata1, ".test-index-2", indexMetadata2, BACKING_INDEX_NAME, dsIndexMetadata)) + .build() + ).build(); + return clusterState; + } + + private static ClusterState getClusterStateWithFailedMigration(String failedIndexName) { + SingleFeatureMigrationResult migrationResult = SingleFeatureMigrationResult.failure(failedIndexName, new Exception()); + FeatureMigrationResults featureMigrationResults = new FeatureMigrationResults(Map.of(FEATURE_NAME, migrationResult)); + + ClusterState initialState = getClusterState(); + return ClusterState.builder(initialState) + .metadata(Metadata.builder(initialState.metadata()).putCustom(FeatureMigrationResults.TYPE, featureMigrationResults).build()) + .build(); + } +} diff --git a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/FeatureMigrationResultsTests.java similarity index 91% rename from server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/FeatureMigrationResultsTests.java index 8eee57b6d9419..3cbbfb062ea87 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/FeatureMigrationResultsTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.Diff; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsTests.java similarity index 78% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsTests.java index c9f2102dd446c..bc111b44aaa9f 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsXContentTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsXContentTests.java similarity index 70% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsXContentTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsXContentTests.java index b66ba693d9aad..bedda2b7b617a 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsXContentTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsXContentTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateTests.java similarity index 89% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateTests.java index 427d4d805fea0..dff6cfcdd19ac 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateXContentTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateXContentTests.java similarity index 76% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateXContentTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateXContentTests.java index a5be76f9a18a3..40f635ac4b09d 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateXContentTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateXContentTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportActionTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportActionTests.java index 99e1031dec3a2..f73f5e5d725e0 100644 --- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportActionTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportActionTests.java @@ -10,6 +10,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -29,9 +33,16 @@ import org.mockito.MockitoAnnotations; import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ReindexDataStreamIndexTransportActionTests extends ESTestCase { @@ -111,7 +122,10 @@ public void testReindexIncludesRateLimit() { ) ); - doNothing().when(client).execute(eq(ReindexAction.INSTANCE), request.capture(), eq(listener)); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.getNodes()).thenReturn(getTestDiscoveryNodes()); + when(clusterService.state()).thenReturn(clusterState); + doNothing().when(transportService).sendRequest(any(), eq(ReindexAction.NAME), request.capture(), any()); action.reindex(sourceIndex, destIndex, listener, taskId); @@ -136,7 +150,10 @@ public void testReindexIncludesInfiniteRateLimit() { Collections.singleton(ReindexDataStreamIndexTransportAction.REINDEX_MAX_REQUESTS_PER_SECOND_SETTING) ) ); - doNothing().when(client).execute(eq(ReindexAction.INSTANCE), request.capture(), eq(listener)); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.getNodes()).thenReturn(getTestDiscoveryNodes()); + when(clusterService.state()).thenReturn(clusterState); + doNothing().when(transportService).sendRequest(any(), eq(ReindexAction.NAME), request.capture(), any()); action.reindex(sourceIndex, destIndex, listener, taskId); @@ -203,4 +220,142 @@ public void testReindexNegativeRateLimitThrowsError() { e.getMessage() ); } + + public void testRoundRobin() { + /* + * This tests that the action will round-robin through the list of ingest nodes in the cluster. + */ + String sourceIndex = randomAlphanumericOfLength(10); + String destIndex = randomAlphanumericOfLength(10); + AtomicBoolean failed = new AtomicBoolean(false); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(BulkByScrollResponse bulkByScrollResponse) {} + + @Override + public void onFailure(Exception e) { + failed.set(true); + } + }; + TaskId taskId = TaskId.EMPTY_TASK_ID; + + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings( + Settings.EMPTY, + Collections.singleton(ReindexDataStreamIndexTransportAction.REINDEX_MAX_REQUESTS_PER_SECOND_SETTING) + ) + ); + + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.getNodes()).thenReturn(getTestDiscoveryNodes()); + when(clusterService.state()).thenReturn(clusterState); + ArgumentCaptor nodeCaptor = ArgumentCaptor.captor(); + doNothing().when(transportService).sendRequest(nodeCaptor.capture(), eq(ReindexAction.NAME), request.capture(), any()); + + action.reindex(sourceIndex, destIndex, listener, taskId); + DiscoveryNode node1 = nodeCaptor.getValue(); + assertNotNull(node1); + + action.reindex(sourceIndex, destIndex, listener, taskId); + DiscoveryNode node2 = nodeCaptor.getValue(); + assertNotNull(node2); + + int ingestNodeCount = clusterState.getNodes().getIngestNodes().size(); + if (ingestNodeCount > 1) { + assertThat(node1.getName(), not(equalTo(node2.getName()))); + } + + // check that if we keep going we eventually get back to the original node: + DiscoveryNode node = node2; + for (int i = 0; i < ingestNodeCount - 1; i++) { + action.reindex(sourceIndex, destIndex, listener, taskId); + node = nodeCaptor.getValue(); + } + assertNotNull(node); + assertThat(node1.getName(), equalTo(node.getName())); + assertThat(failed.get(), equalTo(false)); + + // make sure the listener gets notified of failure if there are no ingest nodes: + when(clusterState.getNodes()).thenReturn(getTestDiscoveryNodesNoIngest()); + action.reindex(sourceIndex, destIndex, listener, taskId); + assertThat(failed.get(), equalTo(true)); + } + + private DiscoveryNodes getTestDiscoveryNodes() { + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); + boolean nodeHasIngestRole = false; + int nodeCount = randomIntBetween(1, 10); + for (int i = 0; i < nodeCount; i++) { + final DiscoveryNode discoveryNode = new DiscoveryNode( + "test-name-" + i, + "test-id-" + i, + "test-ephemeral-id-" + i, + "test-hostname-" + i, + "test-hostaddr", + buildNewFakeTransportAddress(), + Map.of(), + randomSet( + 1, + 5, + () -> randomFrom( + DiscoveryNodeRole.DATA_ROLE, + DiscoveryNodeRole.INGEST_ROLE, + DiscoveryNodeRole.SEARCH_ROLE, + DiscoveryNodeRole.MASTER_ROLE, + DiscoveryNodeRole.MASTER_ROLE + ) + ), + null, + null + ); + nodeHasIngestRole = nodeHasIngestRole || discoveryNode.getRoles().contains(DiscoveryNodeRole.INGEST_ROLE); + builder.add(discoveryNode); + } + if (nodeHasIngestRole == false) { + final DiscoveryNode discoveryNode = new DiscoveryNode( + "test-name-" + nodeCount, + "test-id-" + nodeCount, + "test-ephemeral-id-" + nodeCount, + "test-hostname-" + nodeCount, + "test-hostaddr", + buildNewFakeTransportAddress(), + Map.of(), + Set.of(DiscoveryNodeRole.INGEST_ROLE), + null, + null + ); + builder.add(discoveryNode); + } + return builder.build(); + } + + private DiscoveryNodes getTestDiscoveryNodesNoIngest() { + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); + int nodeCount = randomIntBetween(0, 10); + for (int i = 0; i < nodeCount; i++) { + final DiscoveryNode discoveryNode = new DiscoveryNode( + "test-name-" + i, + "test-id-" + i, + "test-ephemeral-id-" + i, + "test-hostname-" + i, + "test-hostaddr", + buildNewFakeTransportAddress(), + Map.of(), + randomSet( + 1, + 4, + () -> randomFrom( + DiscoveryNodeRole.DATA_ROLE, + DiscoveryNodeRole.SEARCH_ROLE, + DiscoveryNodeRole.MASTER_ROLE, + DiscoveryNodeRole.MASTER_ROLE + ) + ), + null, + null + ); + builder.add(discoveryNode); + } + return builder.build(); + } } diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/MachineLearningPackageLoader.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/MachineLearningPackageLoader.java index a63d911e9d40d..e33fe677179d8 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/MachineLearningPackageLoader.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/MachineLearningPackageLoader.java @@ -109,7 +109,7 @@ public List getBootstrapChecks() { @Override public BootstrapCheckResult check(BootstrapContext context) { try { - validateModelRepository(MODEL_REPOSITORY.get(context.settings()), context.environment().configFile()); + validateModelRepository(MODEL_REPOSITORY.get(context.settings()), context.environment().configDir()); } catch (Exception e) { return BootstrapCheckResult.failure( "Found an invalid configuration for xpack.ml.model_repository. " diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java index 4d40ec7880fb3..90233d7a343bc 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java @@ -153,7 +153,7 @@ synchronized boolean handleDownloadInProgress( ModelDownloadTask inProgress = null; for (var task : tasks) { - if (description.equals(task.getDescription()) && task instanceof ModelDownloadTask downloadTask) { + if (task instanceof ModelDownloadTask downloadTask && (description.equals(downloadTask.getDescription()))) { inProgress = downloadTask; break; } diff --git a/x-pack/plugin/ml-package-loader/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/ml-package-loader/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..df557f9944253 --- /dev/null +++ b/x-pack/plugin/ml-package-loader/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +ALL-UNNAMED: + - outbound_network diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index d690824691c53..ada71e33f7c8e 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -94,7 +94,7 @@ dependencies { } def mlCppVersion(){ - return (project.gradle.parent != null && buildParams.isSnapshotBuild() == false) ? + return (project.gradle.parent != null && buildParams.snapshotBuild == false) ? (project.version + "-SNAPSHOT") : project.version; } diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index 0869ae394d3de..d18f6da13cad2 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -258,4 +258,5 @@ testClusters.configureEach { user username: "no_ml", password: "x-pack-test-password", role: "minimal" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' + systemProperty 'es.queryable_built_in_roles_enabled', 'false' } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java index 0effe5349d43a..96ac7a036655f 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; +import org.elasticsearch.xpack.core.ml.action.PutFilterAction; import org.elasticsearch.xpack.core.ml.action.UpdateFilterAction; import org.elasticsearch.xpack.core.ml.annotations.Annotation; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -98,7 +99,7 @@ public void testCondition() throws Exception { // push the data for the first half buckets postData(job.getId(), joinBetween(0, data.size() / 2, data)); - closeJob(job.getId()); + flushJob(job.getId(), true); List records = getRecords(job.getId()); // remove records that are not anomalies @@ -116,18 +117,35 @@ public void testCondition() throws Exception { JobUpdate.Builder update = new JobUpdate.Builder(job.getId()); update.setDetectorUpdates(Arrays.asList(new JobUpdate.DetectorUpdate(0, null, Arrays.asList(newRule)))); updateJob(job.getId(), update.build()); + // Wait until the notification that the job was updated is indexed + assertBusy( + () -> assertResponse( + prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).setSize(1) + .addSort("timestamp", SortOrder.DESC) + .setQuery( + QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("job_id", job.getId())) + .filter(QueryBuilders.termQuery("level", "info")) + ), + searchResponse -> { + SearchHit[] hits = searchResponse.getHits().getHits(); + assertThat(hits.length, equalTo(1)); + assertThat((String) hits[0].getSourceAsMap().get("message"), containsString("Job updated: [detectors]")); + } + ) + ); } // push second half - openJob(job.getId()); postData(job.getId(), joinBetween(data.size() / 2, data.size(), data)); - closeJob(job.getId()); + flushJob(job.getId(), true); GetRecordsAction.Request recordsAfterFirstHalf = new GetRecordsAction.Request(job.getId()); recordsAfterFirstHalf.setStart(String.valueOf(firstRecordTimestamp + 1)); records = getRecords(recordsAfterFirstHalf); assertThat("records were " + records, (int) (records.stream().filter(r -> r.getProbability() < 0.01).count()), equalTo(1)); assertThat(records.get(0).getByFieldValue(), equalTo("low")); + closeJob(job.getId()); } public void testScope() throws Exception { @@ -242,7 +260,7 @@ public void testScope() throws Exception { closeJob(job.getId()); } - public void testScopeAndCondition() throws IOException { + public void testScopeAndCondition() throws Exception { // We have 2 IPs and they're both safe-listed. List ips = Arrays.asList("111.111.111.111", "222.222.222.222"); MlFilter safeIps = MlFilter.builder("safe_ips").setItems(ips).build(); @@ -298,11 +316,112 @@ public void testScopeAndCondition() throws IOException { } postData(job.getId(), joinBetween(0, data.size(), data)); - closeJob(job.getId()); + flushJob(job.getId(), true); List records = getRecords(job.getId()); assertThat(records.size(), equalTo(1)); assertThat(records.get(0).getOverFieldValue(), equalTo("222.222.222.222")); + + // Remove "111.111.111.111" from the "safe_ips" filter + List addedIps = Arrays.asList(); + List removedIps = Arrays.asList("111.111.111.111"); + PutFilterAction.Response updatedFilter = updateMlFilter("safe_ips", addedIps, removedIps); + // Wait until the notification that the filter was updated is indexed + assertBusy( + () -> assertResponse( + prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).setSize(1) + .addSort("timestamp", SortOrder.DESC) + .setQuery( + QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("job_id", job.getId())) + .filter(QueryBuilders.termQuery("level", "info")) + ), + searchResponse -> { + SearchHit[] hits = searchResponse.getHits().getHits(); + assertThat(hits.length, equalTo(1)); + assertThat( + (String) hits[0].getSourceAsMap().get("message"), + containsString("Filter [safe_ips] has been modified; removed items: ['111.111.111.111']") + ); + } + ) + ); + MlFilter updatedSafeIps = MlFilter.builder("safe_ips").setItems(Arrays.asList("222.222.222.222")).build(); + assertThat(updatedFilter.getFilter(), equalTo(updatedSafeIps)); + + data.clear(); + // Now send anomalous count of 9 for 111.111.111.111 + for (int i = 0; i < 9; i++) { + data.add(createIpRecord(timestamp, "111.111.111.111")); + } + + // Some more normal buckets + for (int bucket = 0; bucket < 3; bucket++) { + for (String ip : ips) { + data.add(createIpRecord(timestamp, ip)); + } + timestamp += TimeValue.timeValueHours(1).getMillis(); + } + + postData(job.getId(), joinBetween(0, data.size(), data)); + flushJob(job.getId(), true); + + records = getRecords(job.getId()); + assertThat(records.size(), equalTo(2)); + assertThat(records.get(0).getOverFieldValue(), equalTo("222.222.222.222")); + assertThat(records.get(1).getOverFieldValue(), equalTo("111.111.111.111")); + + { + // Update detection rules such that it now applies only to actual values > 10.0 + DetectionRule newRule = new DetectionRule.Builder( + Arrays.asList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 10.0)) + ).build(); + JobUpdate.Builder update = new JobUpdate.Builder(job.getId()); + update.setDetectorUpdates(Arrays.asList(new JobUpdate.DetectorUpdate(0, null, Arrays.asList(newRule)))); + updateJob(job.getId(), update.build()); + // Wait until the notification that the job was updated is indexed + assertBusy( + () -> assertResponse( + prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).setSize(1) + .addSort("timestamp", SortOrder.DESC) + .setQuery( + QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("job_id", job.getId())) + .filter(QueryBuilders.termQuery("level", "info")) + ), + searchResponse -> { + SearchHit[] hits = searchResponse.getHits().getHits(); + assertThat(hits.length, equalTo(1)); + assertThat((String) hits[0].getSourceAsMap().get("message"), containsString("Job updated: [detectors]")); + } + ) + ); + } + + data.clear(); + // Now send anomalous count of 10 for 222.222.222.222 + for (int i = 0; i < 10; i++) { + data.add(createIpRecord(timestamp, "222.222.222.222")); + } + + // Some more normal buckets + for (int bucket = 0; bucket < 3; bucket++) { + for (String ip : ips) { + data.add(createIpRecord(timestamp, ip)); + } + timestamp += TimeValue.timeValueHours(1).getMillis(); + } + + postData(job.getId(), joinBetween(0, data.size(), data)); + + closeJob(job.getId()); + + // The anomalous records should not have changed. + records = getRecords(job.getId()); + assertThat(records.size(), equalTo(2)); + assertThat(records.get(0).getOverFieldValue(), equalTo("222.222.222.222")); + assertThat(records.get(1).getOverFieldValue(), equalTo("111.111.111.111")); + } public void testForceTimeShiftAction() throws Exception { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java index e8acc37e0e153..3654f7e17805f 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java @@ -124,7 +124,7 @@ protected StopDataFrameAnalyticsAction.Response forceStopAnalytics(String id) { } protected void waitUntilAnalyticsIsStopped(String id) throws Exception { - waitUntilAnalyticsIsStopped(id, TimeValue.timeValueSeconds(60)); + waitUntilAnalyticsIsStopped(id, TimeValue.timeValueSeconds(180)); } protected void waitUntilAnalyticsIsStopped(String id, TimeValue waitTime) throws Exception { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index d18b6b6cf9ab6..54969b656af31 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -79,6 +79,7 @@ import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.core.ml.action.UpdateFilterAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; import org.elasticsearch.xpack.core.ml.inference.ModelAliasMetadata; @@ -311,6 +312,13 @@ protected PutFilterAction.Response putMlFilter(MlFilter filter) { return client().execute(PutFilterAction.INSTANCE, new PutFilterAction.Request(filter)).actionGet(); } + protected PutFilterAction.Response updateMlFilter(String filterId, List addItems, List removeItems) { + UpdateFilterAction.Request request = new UpdateFilterAction.Request(filterId); + request.setAddItems(addItems); + request.setRemoveItems(removeItems); + return client().execute(UpdateFilterAction.INSTANCE, request).actionGet(); + } + protected static List fetchAllAuditMessages(String jobId) throws Exception { RefreshRequest refreshRequest = new RefreshRequest(NotificationsIndex.NOTIFICATIONS_INDEX); BroadcastResponse refreshResponse = client().execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index 27bce6747b32f..05bd4eeb48a16 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; @@ -37,9 +38,12 @@ import org.elasticsearch.xpack.core.ml.utils.MlStrings; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; +import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.has6DigitSuffix; /** * Rollover the various .ml-anomalies result indices @@ -108,6 +112,14 @@ public void runUpdate(ClusterState latestState) { continue; } + // Check if this index has already been rolled over + String latestIndex = latestIndexMatchingBaseName(index, expressionResolver, latestState); + + if (index.equals(latestIndex) == false) { + logger.debug("index [{}] will not be rolled over as there is a later index [{}]", index, latestIndex); + continue; + } + PlainActionFuture updated = new PlainActionFuture<>(); rollAndUpdateAliases(latestState, index, updated); try { @@ -137,7 +149,7 @@ public void runUpdate(ClusterState latestState) { private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { // Create an alias specifically for rolling over. - // The ml-anomalies index has aliases for each job anyone + // The ml-anomalies index has aliases for each job, any // of which could be used but that means one alias is // treated differently. // Using a `.` in the alias name avoids any conflicts @@ -163,9 +175,19 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio } private void rollover(String alias, @Nullable String newIndexName, ActionListener listener) { - client.admin().indices().rolloverIndex(new RolloverRequest(alias, newIndexName), listener.delegateFailure((l, response) -> { - l.onResponse(response.getNewIndex()); - })); + client.admin() + .indices() + .rolloverIndex( + new RolloverRequest(alias, newIndexName), + ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { + if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { + // The destination index already exists possibly because it has been rolled over already. + listener.onResponse(alreadyExistsException.getIndex().getName()); + } else { + listener.onFailure(e); + } + }) + ); } private void createAliasForRollover(String indexName, String aliasName, ActionListener listener) { @@ -232,4 +254,41 @@ static boolean isAnomaliesReadAlias(String aliasName) { // which is not a valid job id. return MlStrings.isValidId(jobIdPart); } + + /** + * Strip any suffix from the index name and find any other indices + * that match the base name. Then return the latest index from the + * matching ones. + * + * @param index The index to check + * @param expressionResolver The expression resolver + * @param latestState The latest cluster state + * @return The latest index that matches the base name of the given index + */ + static String latestIndexMatchingBaseName(String index, IndexNameExpressionResolver expressionResolver, ClusterState latestState) { + String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index) + ? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length()) + : index; + + String[] matching = expressionResolver.concreteIndexNames( + latestState, + IndicesOptions.lenientExpandOpenHidden(), + baseIndexName + "*" + ); + + // This should never happen + assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]"; + if (matching.length == 0) { + return index; + } + + // Exclude indices that start with the same base name but are a different index + // e.g. .ml-anomalies-foobar should not be included when the index name is + // .ml-anomalies-foo + String[] filtered = Arrays.stream(matching).filter(i -> { + return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length()); + }).toArray(String[]::new); + + return MlIndexAndAlias.latestIndex(filtered); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAutoUpdateService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAutoUpdateService.java index 05c4d70e013e9..87c4b3fd63303 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAutoUpdateService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAutoUpdateService.java @@ -66,7 +66,6 @@ public void clusterChanged(ClusterChangedEvent event) { .filter(action -> action.isAbleToRun(latestState)) .filter(action -> currentlyUpdating.add(action.getName())) .toList(); - // TODO run updates serially threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) .execute(() -> toRun.forEach((action) -> this.runUpdate(action, latestState))); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java index 45c7261cab381..c5ab84d277a0d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java @@ -213,7 +213,8 @@ protected void masterOperation(Task task, Request request, ClusterState state, A && (JobState.CLOSED.equals(MlTasks.getJobState(request.getJobId(), customMetadata)) == false)) { listener.onFailure( ExceptionsHelper.conflictStatusException( - "Cannot upgrade snapshot [{}] for job [{}] as it is the current primary job snapshot and the job's state is [{}]", + "Cannot upgrade snapshot [{}] for job [{}] as it is the current primary job snapshot and the job's state is [{}]. " + + "Please close the job before upgrading the snapshot.", request.getSnapshotId(), request.getJobId(), MlTasks.getJobState(request.getJobId(), customMetadata) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java index 5e24393be0a22..6e377770ed0ef 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java @@ -149,7 +149,7 @@ private void createNativeProcess( ProcessPipes processPipes ) { AnalyticsBuilder analyticsBuilder = new AnalyticsBuilder( - env::tmpFile, + env::tmpDir, nativeController, processPipes, analyticsProcessConfig, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeMemoryUsageEstimationProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeMemoryUsageEstimationProcessFactory.java index 8d106be13882d..99adf6b6d506d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeMemoryUsageEstimationProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeMemoryUsageEstimationProcessFactory.java @@ -116,7 +116,7 @@ private void createNativeProcess( ProcessPipes processPipes ) { AnalyticsBuilder analyticsBuilder = new AnalyticsBuilder( - env::tmpFile, + env::tmpDir, nativeController, processPipes, analyticsProcessConfig, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java index 16ec3ee9b468c..a6f4b9be2f992 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; +import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata; import org.elasticsearch.xpack.ml.MachineLearning; @@ -212,6 +213,7 @@ Collection observeDouble(Function deploymentIdsWithInFlightScaleFromZeroRequests = new ConcurrentSkipListSet<>(); private final Map lastWarningMessages = new ConcurrentHashMap<>(); @@ -223,7 +225,17 @@ public AdaptiveAllocationsScalerService( MeterRegistry meterRegistry, boolean isNlpEnabled ) { - this(threadPool, clusterService, client, inferenceAuditor, meterRegistry, isNlpEnabled, DEFAULT_TIME_INTERVAL_SECONDS); + this( + threadPool, + clusterService, + client, + inferenceAuditor, + meterRegistry, + isNlpEnabled, + DEFAULT_TIME_INTERVAL_SECONDS, + SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS, + SCALE_UP_COOLDOWN_TIME_MILLIS + ); } // visible for testing @@ -234,7 +246,9 @@ public AdaptiveAllocationsScalerService( InferenceAuditor inferenceAuditor, MeterRegistry meterRegistry, boolean isNlpEnabled, - int timeIntervalSeconds + int timeIntervalSeconds, + long scaleToZeroAfterNoRequestsSeconds, + long scaleUpCooldownTimeMillis ) { this.threadPool = threadPool; this.clusterService = clusterService; @@ -243,6 +257,8 @@ public AdaptiveAllocationsScalerService( this.meterRegistry = meterRegistry; this.isNlpEnabled = isNlpEnabled; this.timeIntervalSeconds = timeIntervalSeconds; + this.scaleToZeroAfterNoRequestsSeconds = scaleToZeroAfterNoRequestsSeconds; + this.scaleUpCooldownTimeMillis = scaleUpCooldownTimeMillis; lastInferenceStatsByDeploymentAndNode = new HashMap<>(); lastInferenceStatsTimestampMillis = null; @@ -250,7 +266,6 @@ public AdaptiveAllocationsScalerService( scalers = new HashMap<>(); metrics = new Metrics(); busy = new AtomicBoolean(false); - scaleToZeroAfterNoRequestsSeconds = SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS; } public synchronized void start() { @@ -374,6 +389,9 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo Map recentStatsByDeployment = new HashMap<>(); Map numberOfAllocations = new HashMap<>(); + // Check for recent scale ups in the deployment stats, because a different node may have + // caused a scale up when an inference request arrives and there were zero allocations. + Set hasRecentObservedScaleUp = new HashSet<>(); for (AssignmentStats assignmentStats : statsResponse.getStats().results()) { String deploymentId = assignmentStats.getDeploymentId(); @@ -399,6 +417,12 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo (key, value) -> value == null ? recentStats : value.add(recentStats) ); } + if (nodeStats.getRoutingState() != null && nodeStats.getRoutingState().getState() == RoutingState.STARTING) { + hasRecentObservedScaleUp.add(deploymentId); + } + if (nodeStats.getStartTime() != null && now < nodeStats.getStartTime().toEpochMilli() + scaleUpCooldownTimeMillis) { + hasRecentObservedScaleUp.add(deploymentId); + } } } @@ -414,9 +438,12 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo Integer newNumberOfAllocations = adaptiveAllocationsScaler.scale(); if (newNumberOfAllocations != null) { Long lastScaleUpTimeMillis = lastScaleUpTimesMillis.get(deploymentId); + // hasRecentScaleUp indicates whether this service has recently scaled up the deployment. + // hasRecentObservedScaleUp indicates whether a deployment recently has started, + // potentially triggered by another node. + boolean hasRecentScaleUp = lastScaleUpTimeMillis != null && now < lastScaleUpTimeMillis + scaleUpCooldownTimeMillis; if (newNumberOfAllocations < numberOfAllocations.get(deploymentId) - && lastScaleUpTimeMillis != null - && now < lastScaleUpTimeMillis + SCALE_UP_COOLDOWN_TIME_MILLIS) { + && (hasRecentScaleUp || hasRecentObservedScaleUp.contains(deploymentId))) { logger.debug("adaptive allocations scaler: skipping scaling down [{}] because of recent scaleup.", deploymentId); continue; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java index 4a9d65481d412..c18400bab2a77 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java @@ -239,8 +239,7 @@ public ModelLoadingService( this.licenseState = licenseState; } - // for testing - String getModelId(String modelIdOrAlias) { + public String getModelId(String modelIdOrAlias) { return modelAliasToId.getOrDefault(modelIdOrAlias, modelIdOrAlias); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankService.java index bec162d141eba..977140b8695a1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankService.java @@ -101,7 +101,7 @@ public void loadLocalModel(String modelId, ActionListener listener) */ public void loadLearningToRankConfig(String modelId, Map params, ActionListener listener) { trainedModelProvider.getTrainedModel( - modelId, + modelLoadingService.getModelId(modelId), GetTrainedModelsAction.Includes.all(), null, ActionListener.wrap(trainedModelConfig -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java index 08c141c0858ca..56eb86026935c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java @@ -15,7 +15,6 @@ import org.apache.lucene.search.Weight; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -25,11 +24,11 @@ * respective feature name. */ public class QueryFeatureExtractor implements FeatureExtractor { - private final List featureNames; private final List weights; - private final List scorers; - private DisjunctionDISIApproximation rankerIterator; + + private final DisiPriorityQueue subScorers; + private DisjunctionDISIApproximation approximation; public QueryFeatureExtractor(List featureNames, List weights) { if (featureNames.size() != weights.size()) { @@ -37,40 +36,40 @@ public QueryFeatureExtractor(List featureNames, List weights) { } this.featureNames = featureNames; this.weights = weights; - this.scorers = new ArrayList<>(weights.size()); + this.subScorers = new DisiPriorityQueue(weights.size()); } @Override public void setNextReader(LeafReaderContext segmentContext) throws IOException { - DisiPriorityQueue disiPriorityQueue = new DisiPriorityQueue(weights.size()); - scorers.clear(); - for (Weight weight : weights) { + subScorers.clear(); + for (int i = 0; i < weights.size(); i++) { + var weight = weights.get(i); if (weight == null) { - scorers.add(null); continue; } Scorer scorer = weight.scorer(segmentContext); if (scorer != null) { - disiPriorityQueue.add(new DisiWrapper(scorer)); + subScorers.add(new FeatureDisiWrapper(scorer, featureNames.get(i))); } - scorers.add(scorer); } - - rankerIterator = disiPriorityQueue.size() > 0 ? new DisjunctionDISIApproximation(disiPriorityQueue) : null; + approximation = subScorers.size() > 0 ? new DisjunctionDISIApproximation(subScorers) : null; } @Override public void addFeatures(Map featureMap, int docId) throws IOException { - if (rankerIterator == null) { + if (approximation == null || approximation.docID() > docId) { return; } - - rankerIterator.advance(docId); - for (int i = 0; i < featureNames.size(); i++) { - Scorer scorer = scorers.get(i); - // Do we have a scorer, and does it match the provided document? - if (scorer != null && scorer.docID() == docId) { - featureMap.put(featureNames.get(i), scorer.score()); + if (approximation.docID() < docId) { + approximation.advance(docId); + } + if (approximation.docID() != docId) { + return; + } + var w = (FeatureDisiWrapper) subScorers.topList(); + for (; w != null; w = (FeatureDisiWrapper) w.next) { + if (w.twoPhaseView == null || w.twoPhaseView.matches()) { + featureMap.put(w.featureName, w.scorer.score()); } } } @@ -80,4 +79,12 @@ public List featureNames() { return featureNames; } + private static class FeatureDisiWrapper extends DisiWrapper { + final String featureName; + + FeatureDisiWrapper(Scorer scorer, String featureName) { + super(scorer); + this.featureName = featureName; + } + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 9887152c6f311..139ae471bc388 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -605,7 +605,12 @@ private void auditFilterChanges(String jobId, String filterId, Set added private static void appendCommaSeparatedSet(Set items, StringBuilder sb) { sb.append("["); - Strings.collectionToDelimitedString(items, ", ", "'", "'", sb); + if (items.isEmpty() == false) { + // surround each item with single-quotes + sb.append('\''); + Strings.collectionToDelimitedString(items, "', '", sb); + sb.append('\''); + } sb.append("]"); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index f9e4e62e4e3bc..96ad8c037c39a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -125,6 +125,7 @@ import org.elasticsearch.xpack.core.ml.stats.ForecastStats; import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.security.support.Exceptions; import org.elasticsearch.xpack.ml.job.categorization.GrokPatternCreator; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; @@ -305,11 +306,15 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()); String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias(job.getId()); String tempIndexName = job.getInitialResultsIndexName(); + // Find all indices starting with this name and pick the latest one + String[] concreteIndices = resolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), tempIndexName + "*"); + if (concreteIndices.length > 0) { + tempIndexName = MlIndexAndAlias.latestIndex(concreteIndices); + } // Our read/write aliases should point to the concrete index // If the initial index is NOT an alias, either it is already a concrete index, or it does not exist yet if (state.getMetadata().hasAlias(tempIndexName)) { - String[] concreteIndices = resolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), tempIndexName); // SHOULD NOT be closed as in typical call flow checkForLeftOverDocuments already verified this // if it is closed, we bailout and return an error @@ -323,8 +328,8 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen ); return; } - tempIndexName = concreteIndices[0]; } + final String indexName = tempIndexName; ActionListener indexAndMappingsListener = ActionListener.wrap(success -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java index 2d4ea308a6693..9c6d953cee5fb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java @@ -209,9 +209,9 @@ public static Path writeNormalizerInitState(String jobId, String state, Environm // createTempFile has a race condition where it may return the same // temporary file name to different threads if called simultaneously // from multiple threads, hence add the thread ID to avoid this - FileUtils.recreateTempDirectoryIfNeeded(env.tmpFile()); + FileUtils.recreateTempDirectoryIfNeeded(env.tmpDir()); Path stateFile = Files.createTempFile( - env.tmpFile(), + env.tmpDir(), jobId + "_quantiles_" + Thread.currentThread().getId(), QUANTILES_FILE_EXTENSION ); @@ -227,8 +227,8 @@ private void buildScheduledEventsConfig(List command) throws IOException if (scheduledEvents.isEmpty()) { return; } - FileUtils.recreateTempDirectoryIfNeeded(env.tmpFile()); - Path eventsConfigFile = Files.createTempFile(env.tmpFile(), "eventsConfig", JSON_EXTENSION); + FileUtils.recreateTempDirectoryIfNeeded(env.tmpDir()); + Path eventsConfigFile = Files.createTempFile(env.tmpDir(), "eventsConfig", JSON_EXTENSION); filesToDelete.add(eventsConfigFile); List scheduledEventToRuleWriters = scheduledEvents.stream() @@ -252,8 +252,8 @@ private void buildScheduledEventsConfig(List command) throws IOException } private void buildJobConfig(List command) throws IOException { - FileUtils.recreateTempDirectoryIfNeeded(env.tmpFile()); - Path configFile = Files.createTempFile(env.tmpFile(), "config", JSON_EXTENSION); + FileUtils.recreateTempDirectoryIfNeeded(env.tmpDir()); + Path configFile = Files.createTempFile(env.tmpDir(), "config", JSON_EXTENSION); filesToDelete.add(configFile); try ( OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(configFile), StandardCharsets.UTF_8); @@ -271,8 +271,8 @@ private void buildFiltersConfig(List command) throws IOException { if (referencedFilters.isEmpty()) { return; } - FileUtils.recreateTempDirectoryIfNeeded(env.tmpFile()); - Path filtersConfigFile = Files.createTempFile(env.tmpFile(), "filtersConfig", JSON_EXTENSION); + FileUtils.recreateTempDirectoryIfNeeded(env.tmpDir()); + Path filtersConfigFile = Files.createTempFile(env.tmpDir(), "filtersConfig", JSON_EXTENSION); filesToDelete.add(filtersConfigFile); try ( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java index d69acab30451a..3af068bf8e444 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java @@ -12,19 +12,27 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeState; import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeTaskState; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -44,9 +52,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -153,6 +159,55 @@ synchronized void start() { executor.execute(); } + private void removeDuplicateModelSnapshotDoc(Consumer runAfter) { + String snapshotDocId = jobId + "_model_snapshot_" + snapshotId; + client.prepareSearch(AnomalyDetectorsIndex.jobResultsIndexPattern()) + .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(snapshotDocId))) + .setSize(2) + .addSort(ModelSnapshot.MIN_VERSION.getPreferredName(), org.elasticsearch.search.sort.SortOrder.ASC) + .execute(ActionListener.wrap(searchResponse -> { + if (searchResponse.getHits().getTotalHits().value > 1) { + deleteOlderSnapshotDoc(searchResponse, runAfter); + } else { + onFinish.accept(null); + } + }, e -> { + logger.warn(() -> format("[%s] [%s] error during search for model snapshot documents", jobId, snapshotId), e); + onFinish.accept(null); + })); + } + + private void deleteOlderSnapshotDoc(SearchResponse searchResponse, Consumer runAfter) { + SearchHit firstHit = searchResponse.getHits().getAt(0); + logger.debug(() -> format("[%s] deleting duplicate model snapshot doc [%s]", jobId, firstHit.getId())); + client.prepareDelete() + .setIndex(firstHit.getIndex()) + .setId(firstHit.getId()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(ActionListener.runAfter(ActionListener.wrap(deleteResponse -> { + if ((deleteResponse.getResult() == DocWriteResponse.Result.DELETED) == false) { + logger.warn( + () -> format( + "[%s] [%s] failed to delete old snapshot [%s] result document, document not found", + jobId, + snapshotId, + ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName() + ) + ); + } + }, e -> { + logger.warn( + () -> format( + "[%s] [%s] failed to delete old snapshot [%s] result document", + jobId, + snapshotId, + ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName() + ), + e + ); + }), () -> runAfter.accept(null))); + } + void setTaskToFailed(String reason, ActionListener> listener) { SnapshotUpgradeTaskState taskState = new SnapshotUpgradeTaskState(SnapshotUpgradeState.FAILED, task.getAllocationId(), reason); task.updatePersistentTaskState(taskState, ActionListener.wrap(listener::onResponse, f -> { @@ -259,7 +314,7 @@ void restoreState() { logger.error(() -> format("[%s] [%s] failed to write old state", jobId, snapshotId), e); setTaskToFailed( "Failed to write old state due to: " + e.getMessage(), - ActionListener.wrap(t -> shutdown(e), f -> shutdown(e)) + ActionListener.running(() -> shutdownWithFailure(e)) ); return; } @@ -273,7 +328,7 @@ void restoreState() { logger.error(() -> format("[%s] [%s] failed to flush after writing old state", jobId, snapshotId), e); nextStep = () -> setTaskToFailed( "Failed to flush after writing old state due to: " + e.getMessage(), - ActionListener.wrap(t -> shutdown(e), f -> shutdown(e)) + ActionListener.running(() -> shutdownWithFailure(e)) ); } else { logger.debug( @@ -295,7 +350,7 @@ private void requestStateWrite() { new SnapshotUpgradeTaskState(SnapshotUpgradeState.SAVING_NEW_STATE, task.getAllocationId(), ""), ActionListener.wrap(readingNewState -> { if (continueRunning.get() == false) { - shutdown(null); + shutdownWithFailure(null); return; } submitOperation(() -> { @@ -310,12 +365,12 @@ private void requestStateWrite() { // Execute callback in the UTILITY thread pool, as the current thread in the callback will be one in the // autodetectWorkerExecutor. Trying to run the callback in that executor will cause a dead lock as that // executor has a single processing queue. - (aVoid, e) -> threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> shutdown(e)) + (aVoid, e) -> threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> handlePersistingState(e)) ); logger.debug("[{}] [{}] asked for state to be persisted", jobId, snapshotId); }, f -> { logger.error(() -> format("[%s] [%s] failed to update snapshot upgrader task to started", jobId, snapshotId), f); - shutdown( + shutdownWithFailure( new ElasticsearchStatusException( "Failed to start snapshot upgrade [{}] for job [{}]", RestStatus.INTERNAL_SERVER_ERROR, @@ -378,17 +433,45 @@ private void checkResultsProcessorIsAlive() { } } - void shutdown(Exception e) { + private void handlePersistingState(@Nullable Exception exception) { + assert Thread.currentThread().getName().contains(UTILITY_THREAD_POOL_NAME); + + if (exception != null) { + shutdownWithFailure(exception); + } else { + stopProcess((aVoid, e) -> { + threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> { + autodetectWorkerExecutor.shutdownNow(); + // If there are two snapshot documents in the results indices with the same snapshot id, + // remove the old one. This can happen when the result index has been rolled over and + // the write alias is pointing to the new index. + removeDuplicateModelSnapshotDoc(onFinish); + }); + + }); + } + } + + void shutdownWithFailure(Exception e) { + stopProcess((aVoid, ignored) -> { + threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> { + onFinish.accept(e); + autodetectWorkerExecutor.shutdownNow(); + }); + }); + } + + private void stopProcess(BiConsumer, Exception> runNext) { logger.debug("[{}] [{}] shutdown initiated", jobId, snapshotId); // No point in sending an action to the executor if the process has died if (process.isProcessAlive() == false) { logger.debug("[{}] [{}] process is dead, no need to shutdown", jobId, snapshotId); - onFinish.accept(e); - autodetectWorkerExecutor.shutdownNow(); stateStreamer.cancel(); + runNext.accept(null, null); return; } - Future future = autodetectWorkerExecutor.submit(() -> { + + submitOperation(() -> { try { logger.debug("[{}] [{}] shutdown is now occurring", jobId, snapshotId); if (process.isReady()) { @@ -401,24 +484,10 @@ void shutdown(Exception e) { processor.awaitCompletion(); } catch (IOException | TimeoutException exc) { logger.warn(() -> format("[%s] [%s] failed to shutdown process", jobId, snapshotId), exc); - } finally { - onFinish.accept(e); } logger.debug("[{}] [{}] connection for upgrade has been closed, process is shutdown", jobId, snapshotId); - }); - try { - future.get(); - autodetectWorkerExecutor.shutdownNow(); - } catch (InterruptedException interrupt) { - Thread.currentThread().interrupt(); - } catch (ExecutionException executionException) { - if (processor.isProcessKilled()) { - // In this case the original exception is spurious and highly misleading - throw ExceptionsHelper.conflictStatusException("close snapshot upgrade interrupted by kill request"); - } else { - throw FutureUtils.rethrowExecutionException(executionException); - } - } + return Void.TYPE; + }, runNext); } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AbstractMlAuditor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AbstractMlAuditor.java index 213d3851b3b98..99b03c2725411 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AbstractMlAuditor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AbstractMlAuditor.java @@ -47,7 +47,8 @@ protected AbstractMlAuditor( clusterService.getNodeName(), messageFactory, clusterService, - indexNameExpressionResolver + indexNameExpressionResolver, + clusterService.threadPool().generic() ); clusterService.addListener(event -> { if (event.metadataChanged()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java index df97b39d2e397..594f72398bc9d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java @@ -52,7 +52,7 @@ public NativeStorageProvider(Environment environment, ByteSizeValue minDiskSpace */ public void cleanupLocalTmpStorageInCaseOfUncleanShutdown() { try { - for (Path p : environment.dataFiles()) { + for (Path p : environment.dataDirs()) { IOUtils.rm(p.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER)); } } catch (Exception e) { @@ -79,7 +79,7 @@ public Path tryGetLocalTmpStorage(String uniqueIdentifier, ByteSizeValue request } private Path tryAllocateStorage(String uniqueIdentifier, ByteSizeValue requestedSize) { - for (Path path : environment.dataFiles()) { + for (Path path : environment.dataDirs()) { try { if (getUsableSpace(path) >= requestedSize.getBytes() + minLocalStorageAvailable.getBytes()) { Path tmpDirectory = path.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER).resolve(uniqueIdentifier); @@ -97,7 +97,7 @@ private Path tryAllocateStorage(String uniqueIdentifier, ByteSizeValue requested public boolean localTmpStorageHasEnoughSpace(Path path, ByteSizeValue requestedSize) { Path realPath = path.toAbsolutePath(); - for (Path p : environment.dataFiles()) { + for (Path p : environment.dataDirs()) { try { if (realPath.startsWith(p.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER))) { return getUsableSpace(p) >= requestedSize.getBytes() + minLocalStorageAvailable.getBytes(); @@ -122,7 +122,7 @@ public void cleanupLocalTmpStorage(String uniqueIdentifier) throws IOException { if (path != null) { // do not allow to breakout from the tmp storage provided Path realPath = path.toAbsolutePath(); - for (Path p : environment.dataFiles()) { + for (Path p : environment.dataDirs()) { if (realPath.startsWith(p.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER))) { IOUtils.rm(path); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessPipes.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessPipes.java index 6b09e38b02ea6..9f8378a5b0087 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessPipes.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessPipes.java @@ -94,7 +94,7 @@ public ProcessPipes( ) { this.namedPipeHelper = namedPipeHelper; this.jobId = jobId; - this.tempDir = env.tmpFile(); + this.tempDir = env.tmpDir(); this.timeout = timeout; // The way the pipe names are formed MUST match what is done in the controller main() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java index 6a5e328d7530a..84b00aca81f71 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java @@ -78,7 +78,7 @@ public String getDefaultPipeDirectoryPrefix(Environment env) { // All these factors need to align for everything to work in production. If any changes // are made here then CNamedPipeFactory::defaultPath() in the C++ code will probably // also need to be changed. - return env.tmpFile().toString() + PathUtils.getDefaultFileSystem().getSeparator(); + return env.tmpDir().toString() + PathUtils.getDefaultFileSystem().getSeparator(); } /** diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..5fa5ca8813919 --- /dev/null +++ b/x-pack/plugin/ml/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,12 @@ +org.elasticsearch.ml: + - manage_threads + - files: + - relative_path: mlmodel.conf + relative_to: config + mode: read + - relative_path: "ml-local-data/" + relative_to: data + mode: read_write + - path: \\.\pipe\ + mode: read_write + platform: windows diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index b203d756c3214..b6613db4e819a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import java.util.List; import java.util.Map; @@ -179,6 +180,78 @@ public void testRunUpdate_LegacyIndex() { verifyNoMoreInteractions(client); } + public void testLatestIndexMatchingBaseName_isLatest() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bax", IndexVersion.current(), List.of("job3"))); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + + var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo", + TestIndexNameExpressionResolver.newInstance(), + csBuilder.build() + ); + assertEquals(".ml-anomalies-custom-foo", latest); + } + + public void testLatestIndexMatchingBaseName_hasLater() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000001", IndexVersion.current(), List.of("job3"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000002", IndexVersion.current(), List.of("job4"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000001", IndexVersion.current(), List.of("job5"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000002", IndexVersion.current(), List.of("job6"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000003", IndexVersion.current(), List.of("job7"))); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + var state = csBuilder.build(); + + assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002")); + + var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-foo-000002", latest); + + latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + ".ml-anomalies-custom-baz-000001", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-baz-000003", latest); + } + + public void testLatestIndexMatchingBaseName_CollidingIndexNames() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foodifferent000001", IndexVersion.current(), List.of("job3"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000001", IndexVersion.current(), List.of("job4"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000002", IndexVersion.current(), List.of("job5"))); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + var state = csBuilder.build(); + + var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-foo", latest); + + latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo-notthisone-000001", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-foo-notthisone-000002", latest); + } + private record AliasActionMatcher(String aliasName, String index, IndicesAliasesRequest.AliasActions.Type actionType) { boolean matches(IndicesAliasesRequest.AliasActions aliasAction) { return aliasAction.actionType() == actionType diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java index 79f2a913902df..22fc2e85f056f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java @@ -36,8 +36,8 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Set; @@ -114,7 +114,12 @@ private ClusterState getClusterState(int numAllocations) { return clusterState; } - private GetDeploymentStatsAction.Response getDeploymentStatsResponse(int numAllocations, int inferenceCount, double latency) { + private GetDeploymentStatsAction.Response getDeploymentStatsResponse( + int numAllocations, + int inferenceCount, + double latency, + boolean recentStartup + ) { return new GetDeploymentStatsAction.Response( List.of(), List.of(), @@ -127,7 +132,7 @@ private GetDeploymentStatsAction.Response getDeploymentStatsResponse(int numAllo new AdaptiveAllocationsSettings(true, null, null), 1024, ByteSizeValue.ZERO, - Instant.now(), + Instant.now().minus(1, ChronoUnit.DAYS), List.of( AssignmentStats.NodeStats.forStartedState( DiscoveryNodeUtils.create("node_1"), @@ -140,7 +145,7 @@ private GetDeploymentStatsAction.Response getDeploymentStatsResponse(int numAllo 0, 0, Instant.now(), - Instant.now(), + recentStartup ? Instant.now() : Instant.now().minus(1, ChronoUnit.HOURS), 1, numAllocations, inferenceCount, @@ -156,7 +161,7 @@ private GetDeploymentStatsAction.Response getDeploymentStatsResponse(int numAllo ); } - public void test() throws IOException { + public void test_scaleUp() { // Initialize the cluster with a deployment with 1 allocation. ClusterState clusterState = getClusterState(1); when(clusterService.state()).thenReturn(clusterState); @@ -168,7 +173,9 @@ public void test() throws IOException { inferenceAuditor, meterRegistry, true, - 1 + 1, + 60, + 60_000 ); service.start(); @@ -182,7 +189,7 @@ public void test() throws IOException { doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") var listener = (ActionListener) invocationOnMock.getArguments()[2]; - listener.onResponse(getDeploymentStatsResponse(1, 1, 11.0)); + listener.onResponse(getDeploymentStatsResponse(1, 1, 11.0, false)); return Void.TYPE; }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); @@ -198,7 +205,7 @@ public void test() throws IOException { doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") var listener = (ActionListener) invocationOnMock.getArguments()[2]; - listener.onResponse(getDeploymentStatsResponse(1, 150, 10.0)); + listener.onResponse(getDeploymentStatsResponse(1, 150, 10.0, false)); return Void.TYPE; }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); doAnswer(invocationOnMock -> { @@ -229,7 +236,137 @@ public void test() throws IOException { doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") var listener = (ActionListener) invocationOnMock.getArguments()[2]; - listener.onResponse(getDeploymentStatsResponse(2, 0, 9.0)); + listener.onResponse(getDeploymentStatsResponse(2, 0, 9.0, false)); + return Void.TYPE; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(null); + return Void.TYPE; + }).when(client).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any()); + + safeSleep(1000); + + verify(client, times(1)).threadPool(); + verify(client, times(1)).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + verifyNoMoreInteractions(client, clusterService); + + service.stop(); + } + + public void test_scaleDownToZero_whenNoRequests() { + // Initialize the cluster with a deployment with 1 allocation. + ClusterState clusterState = getClusterState(1); + when(clusterService.state()).thenReturn(clusterState); + + AdaptiveAllocationsScalerService service = new AdaptiveAllocationsScalerService( + threadPool, + clusterService, + client, + inferenceAuditor, + meterRegistry, + true, + 1, + 1, + 2_000 + ); + service.start(); + + verify(clusterService).state(); + verify(clusterService).addListener(same(service)); + verifyNoMoreInteractions(client, clusterService); + reset(client, clusterService); + + // First cycle: 1 inference request, so no need for scaling. + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(getDeploymentStatsResponse(1, 1, 11.0, false)); + return Void.TYPE; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); + + safeSleep(1200); + + verify(client, times(1)).threadPool(); + verify(client, times(1)).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + verifyNoMoreInteractions(client, clusterService); + reset(client, clusterService); + + // Second cycle: 0 inference requests for 1 second, so scale down to 0 allocations. + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(getDeploymentStatsResponse(1, 0, 10.0, false)); + return Void.TYPE; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(null); + return Void.TYPE; + }).when(client).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any()); + + safeSleep(1000); + + verify(client, times(2)).threadPool(); + verify(client, times(1)).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + var updateRequest = new UpdateTrainedModelDeploymentAction.Request("test-deployment"); + updateRequest.setNumberOfAllocations(0); + updateRequest.setIsInternal(true); + verify(client, times(1)).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), eq(updateRequest), any()); + verifyNoMoreInteractions(client, clusterService); + + service.stop(); + } + + public void test_noScaleDownToZero_whenRecentlyScaledUpByOtherNode() { + // Initialize the cluster with a deployment with 1 allocation. + ClusterState clusterState = getClusterState(1); + when(clusterService.state()).thenReturn(clusterState); + + AdaptiveAllocationsScalerService service = new AdaptiveAllocationsScalerService( + threadPool, + clusterService, + client, + inferenceAuditor, + meterRegistry, + true, + 1, + 1, + 2_000 + ); + service.start(); + + verify(clusterService).state(); + verify(clusterService).addListener(same(service)); + verifyNoMoreInteractions(client, clusterService); + reset(client, clusterService); + + // First cycle: 1 inference request, so no need for scaling. + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(getDeploymentStatsResponse(1, 1, 11.0, true)); + return Void.TYPE; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); + + safeSleep(1200); + + verify(client, times(1)).threadPool(); + verify(client, times(1)).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + verifyNoMoreInteractions(client, clusterService); + reset(client, clusterService); + + // Second cycle: 0 inference requests for 1 second, but a recent scale up by another node. + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(getDeploymentStatsResponse(1, 0, 10.0, true)); return Void.TYPE; }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); doAnswer(invocationOnMock -> { @@ -244,6 +381,32 @@ public void test() throws IOException { verify(client, times(1)).threadPool(); verify(client, times(1)).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); verifyNoMoreInteractions(client, clusterService); + reset(client, clusterService); + + // Third cycle: 0 inference requests for 1 second and no recent scale up, so scale down to 0 allocations. + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(getDeploymentStatsResponse(1, 0, 10.0, false)); + return Void.TYPE; + }).when(client).execute(eq(GetDeploymentStatsAction.INSTANCE), eq(new GetDeploymentStatsAction.Request("test-deployment")), any()); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(null); + return Void.TYPE; + }).when(client).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any()); + + safeSleep(1000); + + verify(client, times(2)).threadPool(); + verify(client, times(1)).execute(eq(GetDeploymentStatsAction.INSTANCE), any(), any()); + var updateRequest = new UpdateTrainedModelDeploymentAction.Request("test-deployment"); + updateRequest.setNumberOfAllocations(0); + updateRequest.setIsInternal(true); + verify(client, times(1)).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), eq(updateRequest), any()); + verifyNoMoreInteractions(client, clusterService); service.stop(); } @@ -256,7 +419,9 @@ public void testMaybeStartAllocation() { inferenceAuditor, meterRegistry, true, - 1 + 1, + 60, + 60_000 ); when(client.threadPool()).thenReturn(threadPool); @@ -289,7 +454,9 @@ public void testMaybeStartAllocation_BlocksMultipleRequests() throws Exception { inferenceAuditor, meterRegistry, true, - 1 + 1, + 60, + 60_000 ); var latch = new CountDownLatch(1); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java index 46e54ff3f8c3d..e374517f6e82c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java @@ -42,12 +42,14 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class LearningToRankServiceTests extends ESTestCase { public static final String GOOD_MODEL = "inference-entity-id"; @@ -185,7 +187,10 @@ protected NamedXContentRegistry xContentRegistry() { } private ModelLoadingService mockModelLoadingService() { - return mock(ModelLoadingService.class); + ModelLoadingService modelLoadingService = mock(ModelLoadingService.class); + when(modelLoadingService.getModelId(anyString())).thenAnswer(i -> i.getArgument(0)); + + return modelLoadingService; } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java index 3b25a266bf412..fc935ba1ae0e6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java @@ -12,7 +12,7 @@ import org.apache.lucene.document.IntField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; @@ -43,13 +43,11 @@ public class QueryFeatureExtractorTests extends AbstractBuilderTestCase { - private Directory dir; - private IndexReader reader; - private IndexSearcher searcher; - - private void addDocs(String[] textValues, int[] numberValues) throws IOException { - dir = newDirectory(); - try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir)) { + private IndexReader addDocs(Directory dir, String[] textValues, int[] numberValues) throws IOException { + var config = newIndexWriterConfig(); + // override the merge policy to ensure that docs remain in the same ingestion order + config.setMergePolicy(newLogMergePolicy(random())); + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir, config)) { for (int i = 0; i < textValues.length; i++) { Document doc = new Document(); doc.add(newTextField(TEXT_FIELD_NAME, textValues[i], Field.Store.NO)); @@ -59,98 +57,119 @@ private void addDocs(String[] textValues, int[] numberValues) throws IOException indexWriter.flush(); } } - reader = indexWriter.getReader(); + return indexWriter.getReader(); } - searcher = newSearcher(reader); - searcher.setSimilarity(new ClassicSimilarity()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98127") public void testQueryExtractor() throws IOException { - addDocs( - new String[] { "the quick brown fox", "the slow brown fox", "the grey dog", "yet another string" }, - new int[] { 5, 10, 12, 11 } - ); - QueryRewriteContext ctx = createQueryRewriteContext(); - List queryExtractorBuilders = List.of( - new QueryExtractorBuilder("text_score", QueryProvider.fromParsedQuery(QueryBuilders.matchQuery(TEXT_FIELD_NAME, "quick fox"))) - .rewrite(ctx), - new QueryExtractorBuilder( - "number_score", - QueryProvider.fromParsedQuery(QueryBuilders.rangeQuery(INT_FIELD_NAME).from(12).to(12)) - ).rewrite(ctx), - new QueryExtractorBuilder( - "matching_none", - QueryProvider.fromParsedQuery(QueryBuilders.termQuery(TEXT_FIELD_NAME, "never found term")) - ).rewrite(ctx), - new QueryExtractorBuilder( - "matching_missing_field", - QueryProvider.fromParsedQuery(QueryBuilders.termQuery("missing_text", "quick fox")) - ).rewrite(ctx) - ); - SearchExecutionContext dummySEC = createSearchExecutionContext(); - List weights = new ArrayList<>(); - List featureNames = new ArrayList<>(); - for (QueryExtractorBuilder qeb : queryExtractorBuilders) { - Query q = qeb.query().getParsedQuery().toQuery(dummySEC); - Weight weight = searcher.rewrite(q).createWeight(searcher, ScoreMode.COMPLETE, 1f); - weights.add(weight); - featureNames.add(qeb.featureName()); - } - QueryFeatureExtractor queryFeatureExtractor = new QueryFeatureExtractor(featureNames, weights); - List> extractedFeatures = new ArrayList<>(); - for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) { - int maxDoc = leafReaderContext.reader().maxDoc(); - queryFeatureExtractor.setNextReader(leafReaderContext); - for (int i = 0; i < maxDoc; i++) { - Map featureMap = new HashMap<>(); - queryFeatureExtractor.addFeatures(featureMap, i); - extractedFeatures.add(featureMap); + try (var dir = newDirectory()) { + try ( + var reader = addDocs( + dir, + new String[] { "the quick brown fox", "the slow brown fox", "the grey dog", "yet another string" }, + new int[] { 5, 10, 12, 11 } + ) + ) { + var searcher = newSearcher(reader); + searcher.setSimilarity(new ClassicSimilarity()); + QueryRewriteContext ctx = createQueryRewriteContext(); + List queryExtractorBuilders = List.of( + new QueryExtractorBuilder( + "text_score", + QueryProvider.fromParsedQuery(QueryBuilders.matchQuery(TEXT_FIELD_NAME, "quick fox")) + ).rewrite(ctx), + new QueryExtractorBuilder( + "number_score", + QueryProvider.fromParsedQuery(QueryBuilders.rangeQuery(INT_FIELD_NAME).from(12).to(12)) + ).rewrite(ctx), + new QueryExtractorBuilder( + "matching_none", + QueryProvider.fromParsedQuery(QueryBuilders.termQuery(TEXT_FIELD_NAME, "never found term")) + ).rewrite(ctx), + new QueryExtractorBuilder( + "matching_missing_field", + QueryProvider.fromParsedQuery(QueryBuilders.termQuery("missing_text", "quick fox")) + ).rewrite(ctx), + new QueryExtractorBuilder( + "phrase_score", + QueryProvider.fromParsedQuery(QueryBuilders.matchPhraseQuery(TEXT_FIELD_NAME, "slow brown fox")) + ).rewrite(ctx) + ); + SearchExecutionContext dummySEC = createSearchExecutionContext(); + List weights = new ArrayList<>(); + List featureNames = new ArrayList<>(); + for (QueryExtractorBuilder qeb : queryExtractorBuilders) { + Query q = qeb.query().getParsedQuery().toQuery(dummySEC); + Weight weight = searcher.rewrite(q).createWeight(searcher, ScoreMode.COMPLETE, 1f); + weights.add(weight); + featureNames.add(qeb.featureName()); + } + QueryFeatureExtractor queryFeatureExtractor = new QueryFeatureExtractor(featureNames, weights); + List> extractedFeatures = new ArrayList<>(); + for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) { + int maxDoc = leafReaderContext.reader().maxDoc(); + queryFeatureExtractor.setNextReader(leafReaderContext); + for (int i = 0; i < maxDoc; i++) { + Map featureMap = new HashMap<>(); + queryFeatureExtractor.addFeatures(featureMap, i); + extractedFeatures.add(featureMap); + } + } + assertThat(extractedFeatures, hasSize(4)); + // Should never add features for queries that don't match a document or on documents where the field is missing + for (Map features : extractedFeatures) { + assertThat(features, not(hasKey("matching_none"))); + assertThat(features, not(hasKey("matching_missing_field"))); + } + // First two only match the text field + assertThat(extractedFeatures.get(0), hasEntry("text_score", 1.7135582f)); + assertThat(extractedFeatures.get(0), not(hasKey("number_score"))); + assertThat(extractedFeatures.get(0), not(hasKey("phrase_score"))); + assertThat(extractedFeatures.get(1), hasEntry("text_score", 0.7554128f)); + assertThat(extractedFeatures.get(1), not(hasKey("number_score"))); + assertThat(extractedFeatures.get(1), hasEntry("phrase_score", 2.468971f)); + + // Only matches the range query + assertThat(extractedFeatures.get(2), hasEntry("number_score", 1f)); + assertThat(extractedFeatures.get(2), not(hasKey("text_score"))); + assertThat(extractedFeatures.get(2), not(hasKey("phrase_score"))); + + // No query matches + assertThat(extractedFeatures.get(3), anEmptyMap()); } } - assertThat(extractedFeatures, hasSize(4)); - // Should never add features for queries that don't match a document or on documents where the field is missing - for (Map features : extractedFeatures) { - assertThat(features, not(hasKey("matching_none"))); - assertThat(features, not(hasKey("matching_missing_field"))); - } - // First two only match the text field - assertThat(extractedFeatures.get(0), hasEntry("text_score", 1.7135582f)); - assertThat(extractedFeatures.get(0), not(hasKey("number_score"))); - assertThat(extractedFeatures.get(1), hasEntry("text_score", 0.7554128f)); - assertThat(extractedFeatures.get(1), not(hasKey("number_score"))); - // Only matches the range query - assertThat(extractedFeatures.get(2), hasEntry("number_score", 1f)); - assertThat(extractedFeatures.get(2), not(hasKey("text_score"))); - // No query matches - assertThat(extractedFeatures.get(3), anEmptyMap()); - reader.close(); - dir.close(); } public void testEmptyDisiPriorityQueue() throws IOException { - addDocs( - new String[] { "the quick brown fox", "the slow brown fox", "the grey dog", "yet another string" }, - new int[] { 5, 10, 12, 11 } - ); + try (var dir = newDirectory()) { + var config = newIndexWriterConfig(); + config.setMergePolicy(NoMergePolicy.INSTANCE); + try ( + var reader = addDocs( + dir, + new String[] { "the quick brown fox", "the slow brown fox", "the grey dog", "yet another string" }, + new int[] { 5, 10, 12, 11 } + ) + ) { - // Scorers returned by weights are null - List featureNames = randomList(1, 10, ESTestCase::randomIdentifier); - List weights = Stream.generate(() -> mock(Weight.class)).limit(featureNames.size()).toList(); + var searcher = newSearcher(reader); + searcher.setSimilarity(new ClassicSimilarity()); - QueryFeatureExtractor featureExtractor = new QueryFeatureExtractor(featureNames, weights); + // Scorers returned by weights are null + List featureNames = randomList(1, 10, ESTestCase::randomIdentifier); + List weights = Stream.generate(() -> mock(Weight.class)).limit(featureNames.size()).toList(); - for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) { - int maxDoc = leafReaderContext.reader().maxDoc(); - featureExtractor.setNextReader(leafReaderContext); - for (int i = 0; i < maxDoc; i++) { - Map featureMap = new HashMap<>(); - featureExtractor.addFeatures(featureMap, i); - assertThat(featureMap, anEmptyMap()); + QueryFeatureExtractor featureExtractor = new QueryFeatureExtractor(featureNames, weights); + for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) { + int maxDoc = leafReaderContext.reader().maxDoc(); + featureExtractor.setNextReader(leafReaderContext); + for (int i = 0; i < maxDoc; i++) { + Map featureMap = new HashMap<>(); + featureExtractor.addFeatures(featureMap, i); + assertThat(featureMap, anEmptyMap()); + } + } } } - - reader.close(); - dir.close(); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/NativeStorageProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/NativeStorageProviderTests.java index f2a4add8444bb..22a6ff630f2bc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/NativeStorageProviderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/NativeStorageProviderTests.java @@ -123,7 +123,7 @@ public void testTmpStorageCleanupOnStart() throws IOException { private NativeStorageProvider createNativeStorageProvider(Map paths) throws IOException { Environment environment = mock(Environment.class); - when(environment.dataFiles()).thenReturn(paths.keySet().toArray(new Path[paths.size()])); + when(environment.dataDirs()).thenReturn(paths.keySet().toArray(new Path[paths.size()])); NativeStorageProvider storageProvider = spy(new NativeStorageProvider(environment, ByteSizeValue.ofGb(5))); doAnswer( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelperTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelperTests.java index d09f0cbb59c1b..fc1b5abc04fbb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelperTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelperTests.java @@ -67,7 +67,7 @@ public void testOpenForInputGivenPipeIsRegularFile() throws IOException { Environment env = TestEnvironment.newEnvironment( Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build() ); - Path tempFile = Files.createTempFile(env.tmpFile(), "not a named pipe", null); + Path tempFile = Files.createTempFile(env.tmpDir(), "not a named pipe", null); IOException ioe = ESTestCase.expectThrows( IOException.class, @@ -83,7 +83,7 @@ public void testOpenForOutputGivenPipeIsRegularFile() throws IOException { Environment env = TestEnvironment.newEnvironment( Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build() ); - Path tempFile = Files.createTempFile(env.tmpFile(), "not a named pipe", null); + Path tempFile = Files.createTempFile(env.tmpDir(), "not a named pipe", null); IOException ioe = ESTestCase.expectThrows( IOException.class, diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java index 0605177b2c2e5..6a76d6749489a 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @@ -77,7 +77,7 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry { * writes monitoring data in ECS format as of 8.0. These templates define the ECS schema as well as alias fields for the old monitoring * mappings that point to the corresponding ECS fields. */ - public static final int STACK_MONITORING_REGISTRY_VERSION = 8_00_00_99 + 20; + public static final int STACK_MONITORING_REGISTRY_VERSION = 8_00_00_99 + 21; private static final String STACK_MONITORING_REGISTRY_VERSION_VARIABLE = "xpack.stack.monitoring.template.release.version"; private static final String STACK_TEMPLATE_VERSION = "8"; private static final String STACK_TEMPLATE_VERSION_VARIABLE = "xpack.stack.monitoring.template.version"; diff --git a/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml index d826de8ca8725..27ff2988cdcbe 100644 --- a/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,8 @@ ALL-UNNAMED: - set_https_connection_properties # potentially required by apache.httpcomponents + # the original policy has java.net.SocketPermission "*", "accept,connect" + # but a comment stating it was "needed for multiple server implementations used in tests" + # TODO: this is likely not needed, but including here to be on the safe side until + # we can track down whether it's really needed + - outbound_network + - inbound_network diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 25b4b685ac50f..3ed1e1877d9e9 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; -import org.apache.lucene.backward_codecs.lucene70.Lucene70Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldsConsumer; @@ -18,6 +17,7 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; @@ -27,7 +27,12 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Version; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec; import java.io.IOException; import java.util.ArrayList; @@ -39,55 +44,122 @@ */ public abstract class BWCCodec extends Codec { + private final FieldInfosFormat fieldInfosFormat; + private final SegmentInfoFormat segmentInfosFormat; + private final PostingsFormat postingsFormat; + protected BWCCodec(String name) { super(name); - } - @Override - public NormsFormat normsFormat() { - throw new UnsupportedOperationException(); - } + this.fieldInfosFormat = new FieldInfosFormat() { + final FieldInfosFormat wrappedFormat = originalFieldInfosFormat(); - @Override - public TermVectorsFormat termVectorsFormat() { - throw new UnsupportedOperationException(); - } + @Override + public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) + throws IOException { + return filterFields(wrappedFormat.read(directory, segmentInfo, segmentSuffix, iocontext)); + } - @Override - public KnnVectorsFormat knnVectorsFormat() { - throw new UnsupportedOperationException(); - } + @Override + public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) + throws IOException { + wrappedFormat.write(directory, segmentInfo, segmentSuffix, infos, context); + } + }; + + this.segmentInfosFormat = new SegmentInfoFormat() { + final SegmentInfoFormat wrappedFormat = originalSegmentInfoFormat(); - protected static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) { - return new SegmentInfoFormat() { @Override public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { - return wrap(wrapped.read(directory, segmentName, segmentID, context)); + return wrap(wrappedFormat.read(directory, segmentName, segmentID, context)); } @Override public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException { - wrapped.write(dir, info, ioContext); + wrappedFormat.write(dir, info, ioContext); } }; - } - protected static FieldInfosFormat wrap(FieldInfosFormat wrapped) { - return new FieldInfosFormat() { + this.postingsFormat = new PerFieldPostingsFormat() { @Override - public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) - throws IOException { - return filterFields(wrapped.read(directory, segmentInfo, segmentSuffix, iocontext)); - } - - @Override - public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) - throws IOException { - wrapped.write(directory, segmentInfo, segmentSuffix, infos, context); + public PostingsFormat getPostingsFormatForField(String field) { + throw new UnsupportedOperationException("Old codecs can't be used for writing"); } }; } + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public final SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * This method is never called in practice, as we rewrite field infos to override the info about which features are present in + * the index. Even if norms are present, field info lies about it. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ + @Override + public final NormsFormat normsFormat() { + throw new UnsupportedOperationException(); + } + + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * This method is never called in practice, as we rewrite field infos to override the info about which features are present in + * the index. Even if term vectors are present, field info lies about it. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ + @Override + public final TermVectorsFormat termVectorsFormat() { + throw new UnsupportedOperationException(); + } + + /** + * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}. + * The knn vectors can't be present because it is not supported yet in any of the lucene versions that we support for archive indices. + * + * @return nothing, as this method always throws an exception + * @throws UnsupportedOperationException always thrown to indicate that this method is not supported + */ + @Override + public final KnnVectorsFormat knnVectorsFormat() { + throw new UnsupportedOperationException(); + } + + /** + * Returns the original {@link SegmentInfoFormat} used by this codec. + * This method should be implemented by subclasses to provide the specific + * {@link SegmentInfoFormat} that this codec is intended to use. + * + * @return the original {@link SegmentInfoFormat} used by this codec + */ + protected abstract SegmentInfoFormat originalSegmentInfoFormat(); + + /** + * Returns the original {@link FieldInfosFormat} used by this codec. + * This method should be implemented by subclasses to provide the specific + * {@link FieldInfosFormat} that this codec is intended to use. + * + * @return the original {@link FieldInfosFormat} used by this codec + */ + protected abstract FieldInfosFormat originalFieldInfosFormat(); + // mark all fields as no term vectors, no norms, no payloads, and no vectors. private static FieldInfos filterFields(FieldInfos fieldInfos) { List fieldInfoCopy = new ArrayList<>(fieldInfos.size()); @@ -119,15 +191,14 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { } public static SegmentInfo wrap(SegmentInfo segmentInfo) { - // special handling for Lucene70Codec (which is currently bundled with Lucene) - // Use BWCLucene70Codec instead as that one extends BWCCodec (similar to all other older codecs) - final Codec codec = segmentInfo.getCodec() instanceof Lucene70Codec ? new BWCLucene70Codec() : segmentInfo.getCodec(); + Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec()); + final SegmentInfo segmentInfo1 = new SegmentInfo( segmentInfo.dir, // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed? - org.apache.lucene.util.Version.LATEST, - org.apache.lucene.util.Version.LATEST, + Version.LATEST, + Version.LATEST, segmentInfo.name, segmentInfo.maxDoc(), segmentInfo.getUseCompoundFile(), @@ -142,6 +213,28 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { return segmentInfo1; } + /** + * Returns a backward-compatible codec for the given codec. If the codec is one of the known Lucene 8.x codecs, + * it returns a corresponding read-only backward-compatible codec. Otherwise, it returns the original codec. + * Lucene 8.x codecs are still shipped with the current version of Lucene. + * Earlier codecs we are providing directly they will also be read-only backward-compatible, but they don't require the renaming. + * + * This switch is only for indices created in ES 6.x, later written into in ES 7.x (Lucene 8.x). Indices created + * in ES 7.x can be read directly by ES if marked read-only, without going through archive indices. + */ + private static Codec getBackwardCompatibleCodec(Codec codec) { + if (codec == null) return null; + + return switch (codec.getClass().getSimpleName()) { + case "Lucene70Codec" -> new BWCLucene70Codec(); + case "Lucene80Codec" -> new BWCLucene80Codec(); + case "Lucene84Codec" -> new BWCLucene84Codec(); + case "Lucene86Codec" -> new BWCLucene86Codec(); + case "Lucene87Codec" -> new BWCLucene87Codec(); + default -> codec; + }; + } + /** * In-memory postings format that shows no postings available. */ diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java index 9694c8bf34d67..9ff5cace8ecd0 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java @@ -47,8 +47,7 @@ */ @Deprecated public class Lucene60Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene50SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -89,18 +88,18 @@ public Lucene60Codec(Lucene50StoredFieldsFormat.Mode mode) { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene50SegmentInfoFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java index fc90a3e14b944..6499f8af72bb2 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java @@ -28,6 +28,7 @@ import java.io.IOException; /** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat} * Allows reading metadata only from Lucene 6.0 point format **/ public class Lucene60MetadataOnlyPointsFormat extends PointsFormat { diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java index 2e796a04200fe..8a5ca4acd16cb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java @@ -34,7 +34,10 @@ import java.util.HashMap; import java.util.Map; -/** Reads the metadata of point values previously written with Lucene60PointsWriter */ +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsReader} + * Reads the metadata of point values previously written with Lucene60PointsWriter + */ public final class Lucene60MetadataOnlyPointsReader extends PointsReader { final IndexInput dataIn; final SegmentReadState readState; @@ -105,7 +108,7 @@ public Lucene60MetadataOnlyPointsReader(SegmentReadState readState) throws IOExc int fieldNumber = ent.getKey(); long fp = ent.getValue(); dataIn.seek(fp); - PointValues reader = new MetadataOnlyBKDReader(dataIn); + PointValues reader = new MetadataOnlyBKDReader(dataIn, false); readers.put(fieldNumber, reader); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java index f3ce3ea0755e1..2a43f070b3616 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java @@ -47,7 +47,7 @@ public class MetadataOnlyBKDReader extends PointValues { final int docCount; final int version; - public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { + public MetadataOnlyBKDReader(IndexInput metaIn, boolean isVersionPost86) throws IOException { version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT); final int numDims = metaIn.readVInt(); final int numIndexDims; @@ -85,6 +85,23 @@ public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException { pointCount = metaIn.readVLong(); docCount = metaIn.readVInt(); + + // The pre-8.6 code does not read the following fields that its standard Lucene counterpart does. After experimenting with the + // code, we got to the conclusion that these are the last fields being read, which are not needed in the metadata-only reader, and + // we can safely ignore them when loading the file. Although by coincidence, nothing breaks if we read a couple of VLongs, as long + // as some bytes are available to read. + // + // The extra reads have been introduced to process IndexInput created with Lucene86Codec+, where a new BKD format has been + // introduced. We have stricter checks around the header and footer starting from the 86 formats hence we do need to + // consume all the data input there but not in previous formats. + // + // For correctness, we added version checking here. If and only if, the version is 8.6 or higher, we read the additional fields. + if (isVersionPost86) { + metaIn.readVInt(); + metaIn.readLong(); + // The following fields are not used in this class, but we need to read them to advance the pointer + metaIn.readLong(); + } } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java index 61579d33e41cb..adc1f97c71f12 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java @@ -47,8 +47,7 @@ */ @Deprecated public class Lucene62Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene62SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; @@ -80,18 +79,18 @@ public Lucene62Codec(Lucene50StoredFieldsFormat.Mode mode) { } @Override - public final StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public final FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene62SegmentInfoFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; } @Override diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java index 0e689138acd8f..8648ac859c386 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -11,54 +11,58 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70DocValuesFormat; import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; -import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; +/** + * Implements the Lucene 7.0 index format. Loaded via SPI for indices created/written with Lucene 7.x (Elasticsearch 6.x) mounted + * as archive indices first in Elasticsearch 8.x. Lucene 9.12 retained Lucene70Codec in its classpath which required overriding the + * codec name and version in the segment infos. This codec is still needed after upgrading to Elasticsearch 9.x because its codec + * name has been written to disk. + */ public class BWCLucene70Codec extends BWCCodec { - private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); private final StoredFieldsFormat storedFieldsFormat; - private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene70"); + private final DocValuesFormat defaultDVFormat = new Lucene70DocValuesFormat(); private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { return defaultDVFormat; } }; - private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - throw new IllegalStateException("This codec should only be used for reading, not writing"); - } - }; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + // Needed for SPI loading + @SuppressWarnings("unused") public BWCLucene70Codec() { - super("BWCLucene70Codec"); + this("BWCLucene70Codec"); + } + + protected BWCLucene70Codec(String name) { + super(name); storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); } @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); } @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); } @Override @@ -81,13 +85,8 @@ public final DocValuesFormat docValuesFormat() { return docValuesFormat; } - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; - } - @Override public PointsFormat pointsFormat() { - return new Lucene60MetadataOnlyPointsFormat(); + return pointsFormat; } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java new file mode 100644 index 0000000000000..9537b4e6f7fa0 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene80.Lucene80Codec} + * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0 + * (Elasticsearch [7.0.0-7.5.2]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene80Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat(); + + private final StoredFieldsFormat storedFieldsFormat; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene80Codec() { + super("BWCLucene80Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public final CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public final PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java new file mode 100644 index 0000000000000..6771f4b3130c1 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene84.Lucene84Codec} + * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1 + * (Elasticsearch [7.6.0-7.8.1]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene84Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final DocValuesFormat defaultDVFormat; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat(); + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene84Codec() { + super("BWCLucene84Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.defaultDVFormat = new Lucene80DocValuesFormat(); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene70SegmentInfoFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java new file mode 100644 index 0000000000000..1949285118aed --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86Codec} + * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2 + * (Elasticsearch [7.9.0-7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene86Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + private final DocValuesFormat defaultDVFormat; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene86Codec() { + super("BWCLucene86Codec"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + this.defaultDVFormat = new Lucene80DocValuesFormat(); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java new file mode 100644 index 0000000000000..f7902c5c9e2a0 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java @@ -0,0 +1,56 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.codecs.PointsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat} + * Allows reading metadata only from Lucene 8.6 point format + **/ +public class Lucene86MetadataOnlyPointsFormat extends PointsFormat { + + static final String META_CODEC_NAME = "Lucene86PointsFormatMeta"; + + /** Filename extension for the meta per field */ + public static final String META_EXTENSION = "kdm"; + + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + /** Sole constructor */ + public Lucene86MetadataOnlyPointsFormat() {} + + @Override + public PointsWriter fieldsWriter(SegmentWriteState state) { + throw new UnsupportedOperationException("Old codecs may only be used for reading"); + } + + @Override + public PointsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene86MetadataOnlyPointsReader(state); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java new file mode 100644 index 0000000000000..55671828b4dcd --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java @@ -0,0 +1,121 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86; + +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.ChecksumIndexInput; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsReader} + * Reads the metadata of point values previously written with Lucene86PointsWriter + */ +public final class Lucene86MetadataOnlyPointsReader extends PointsReader { + final SegmentReadState readState; + final Map readers = new HashMap<>(); + + public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException { + this.readState = readState; + + String metaFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Lucene86MetadataOnlyPointsFormat.META_EXTENSION + ); + + boolean success = false; + try { + try ( + ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context) + ) { + Throwable priorE = null; + try { + CodecUtil.checkIndexHeader( + metaIn, + Lucene86MetadataOnlyPointsFormat.META_CODEC_NAME, + Lucene86MetadataOnlyPointsFormat.VERSION_START, + Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + + while (true) { + int fieldNumber = metaIn.readInt(); + if (fieldNumber == -1) { + break; + } else if (fieldNumber < 0) { + throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn); + } + PointValues reader = new MetadataOnlyBKDReader(metaIn, true); + readers.put(fieldNumber, reader); + } + metaIn.readLong(); + metaIn.readLong(); + } catch (Throwable t) { + priorE = t; + } finally { + CodecUtil.checkFooter(metaIn, priorE); + } + } + + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public PointValues getValues(String fieldName) { + FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName); + if (fieldInfo == null) { + throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized"); + } + if (fieldInfo.getPointDimensionCount() == 0) { + throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values"); + } + + return readers.get(fieldInfo.number); + } + + // We only open the metadata field, and do nothing with the other two files (index/data), + // for which Lucene checks integrity but we don't need to. + @Override + public void checkIntegrity() {} + + @Override + public void close() throws IOException { + // Free up heap: + readers.clear(); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java new file mode 100644 index 0000000000000..f461bdee8864d --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; +import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.Lucene86MetadataOnlyPointsFormat; + +/** + * This is a fork of {@link org.apache.lucene.backward_codecs.lucene87.Lucene87Codec} + * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.3 + * (Elasticsearch [7.10.0-7-17.26]), mounted as archive indices in Elasticsearch 8.x / 9.x. + */ +public class BWCLucene87Codec extends BWCCodec { + + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat(); + private final DocValuesFormat defaultDVFormat; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + }; + + private final StoredFieldsFormat storedFieldsFormat; + + // Needed for SPI loading + @SuppressWarnings("unused") + public BWCLucene87Codec() { + super("BWCLucene87Codec"); + this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION); + this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return new Lucene60FieldInfosFormat(); + } + + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return new Lucene86SegmentInfoFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } + + @Override + public PointsFormat pointsFormat() { + return pointsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 6e5205d664f2d..abc3ed8dd5323 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,6 +5,10 @@ # 2.0. # +org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60Codec diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java new file mode 100644 index 0000000000000..219cfa29f13ce --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.elasticsearch.test.ESTestCase; + +/** + * Unit tests for the {@link BWCCodec} class. + */ +public class BWCCodecTests extends ESTestCase { + + private final Codec codec; + + public BWCCodecTests() { + this.codec = new BWCCodec("WrapperCodec") { + @Override + protected SegmentInfoFormat originalSegmentInfoFormat() { + return null; + } + + @Override + protected FieldInfosFormat originalFieldInfosFormat() { + return null; + } + + @Override + public PostingsFormat postingsFormat() { + return null; + } + + @Override + public DocValuesFormat docValuesFormat() { + return null; + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return null; + } + + @Override + public LiveDocsFormat liveDocsFormat() { + return null; + } + + @Override + public CompoundFormat compoundFormat() { + return null; + } + + @Override + public PointsFormat pointsFormat() { + return null; + } + }; + } + + /** + * Tests that the {@link Codec#normsFormat()} method throws an {@link UnsupportedOperationException}. + */ + public void testNormsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::normsFormat); + } + + /** + * Tests that the {@link Codec#termVectorsFormat()} method throws an {@link UnsupportedOperationException}. + */ + public void testTermVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat); + } + + /** + * Tests that the {@link Codec#knnVectorsFormat()} method throws an {@link UnsupportedOperationException}. + */ + public void testKnnVectorsFormatUnsupportedOperation() { + assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat); + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index bf1538b4e5dd8..5674504b4eddc 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -7,17 +7,61 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; +import org.apache.lucene.codecs.Codec; import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; +import java.util.ServiceLoader; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class OldCodecsAvailableTests extends ESTestCase { /** + * This test verifies for each Lucene codec available via SPI; we also provide a corresponding BWC codec counterpart. + * Using a ServiceLoader, we fetch all classes matching the codecPathRegex (this is applied for Lucne8xCodec at the moment). + * For each entry of the returned list, we intend to load the BWC counterpart reflectively. + * * Reminder to add Lucene BWC codecs under {@link org.elasticsearch.xpack.lucene.bwc.codecs} whenever Elasticsearch is upgraded * to the next major Lucene version. */ public void testLuceneBWCCodecsAvailable() { assertEquals("Add Lucene BWC codecs for Elasticsearch version 7", 8, Version.CURRENT.major); + + String codecPathRegex = ".*[\\\\.](Lucene(8[0-9])Codec)"; + Pattern codecPathPattern = Pattern.compile(codecPathRegex); + + String codecClassNameRegex = "Lucene(\\d+)Codec"; + Pattern classNamePattern = Pattern.compile(codecClassNameRegex); + + for (Codec codec : ServiceLoader.load(Codec.class)) { + Matcher codecPathMatcher = codecPathPattern.matcher(codec.getClass().getName()); + if (codecPathMatcher.matches()) { + String pathName = codec.getClass().getName(); + int lastDotIndex = pathName.lastIndexOf('.'); + String className = pathName.substring(lastDotIndex + 1); + + Matcher classNameMatcher = classNamePattern.matcher(className); + if (classNameMatcher.matches()) { + String codecVersion = classNameMatcher.group(1); + String wrappedCodecClassPath = "org.elasticsearch.xpack.lucene.bwc.codecs.lucene" + + codecVersion + + ".BWCLucene" + + codecVersion + + "Codec"; + assertTrue(isClassPresent(wrappedCodecClassPath)); + } + } + } + } + + private static boolean isClassPresent(String className) { + try { + Class.forName(className); + return true; + } catch (ClassNotFoundException e) { + return false; + } } } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java index 0be6d91450eda..5c9b589b4ec38 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/StackTrace.java @@ -184,7 +184,15 @@ static String getFileIDFromStackFrameID(String frameID) { public static StackTrace fromSource(Map source) { String inputFrameIDs = ObjectPath.eval(PATH_FRAME_IDS, source); + if (inputFrameIDs == null) { + // If synthetic source is disabled, fallback to dotted field names. + inputFrameIDs = (String) source.get("Stacktrace.frame.ids"); + } String inputFrameTypes = ObjectPath.eval(PATH_FRAME_TYPES, source); + if (inputFrameTypes == null) { + // If synthetic source is disabled, fallback to dotted field names. + inputFrameTypes = (String) source.get("Stacktrace.frame.types"); + } int countsFrameIDs = inputFrameIDs.length() / BASE64_FRAME_ID_LENGTH; String[] fileIDs = new String[countsFrameIDs]; diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java index 48673d2002170..7935be02e1fd9 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java @@ -636,7 +636,7 @@ public void calculateCO2AndCosts() { if (missingStackTraces.isEmpty() == false) { StringBuilder stringBuilder = new StringBuilder(); - Strings.collectionToDelimitedStringWithLimit(missingStackTraces, ",", "", "", 80, stringBuilder); + Strings.collectionToDelimitedStringWithLimit(missingStackTraces, ",", 80, stringBuilder); log.warn("CO2/cost calculator: missing trace events for StackTraceID [" + stringBuilder + "]."); } } @@ -796,7 +796,12 @@ public void onExecutableDetailsResponse(MultiGetResponse multiGetItemResponses) if (executable.getResponse().isExists()) { // Duplicates are expected as we query multiple indices - do a quick pre-check before we deserialize a response if (executables.containsKey(executable.getId()) == false) { - String fileName = ObjectPath.eval(PATH_FILE_NAME, executable.getResponse().getSource()); + Map source = executable.getResponse().getSource(); + String fileName = ObjectPath.eval(PATH_FILE_NAME, source); + if (fileName == null) { + // If synthetic source is disabled, read from dotted field names. + fileName = (String) source.get("Executable.file.name"); + } if (fileName != null) { executables.putIfAbsent(executable.getId(), fileName); } else { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java index 20e8214ddef34..6fdb835064a28 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.ql.expression; -import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; /** @@ -28,7 +27,7 @@ public NameId() { @Override public int hashCode() { - return Objects.hash(id); + return Long.hashCode(id); } @Override diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/QueryPlan.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/QueryPlan.java index fa38e87612d5d..231a8e00349b2 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/QueryPlan.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/QueryPlan.java @@ -109,8 +109,8 @@ public PlanType transformExpressionsUp(Class typeToken @SuppressWarnings("unchecked") private static Object doTransformExpression(Object arg, Function traversal) { - if (arg instanceof Expression) { - return traversal.apply((Expression) arg); + if (arg instanceof Expression exp) { + return traversal.apply(exp); } // WARNING: if the collection is typed, an incompatible function will be applied to it @@ -119,17 +119,19 @@ private static Object doTransformExpression(Object arg, Function c) { - List transformed = new ArrayList<>(c.size()); + List transformed = null; boolean hasChanged = false; + int i = 0; for (Object e : c) { Object next = doTransformExpression(e, traversal); - if (e.equals(next)) { - // use the initial value - next = e; - } else { - hasChanged = true; + if (e.equals(next) == false) { + if (hasChanged == false) { + hasChanged = true; + transformed = new ArrayList<>(c); + } + transformed.set(i, next); } - transformed.add(next); + i++; } return hasChanged ? transformed : arg; @@ -164,8 +166,8 @@ public void forEachExpressionUp(Class typeToken, Consu @SuppressWarnings("unchecked") private static void doForEachExpression(Object arg, Consumer traversal) { - if (arg instanceof Expression) { - traversal.accept((Expression) arg); + if (arg instanceof Expression exp) { + traversal.accept(exp); } else if (arg instanceof Collection c) { for (Object o : c) { doForEachExpression(o, traversal); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/Node.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/Node.java index cb0233429f323..b51b303eed14b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/Node.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/Node.java @@ -109,7 +109,7 @@ public void forEachPropertyUp(Class typeToken, Consumer rule) protected void forEachProperty(Class typeToken, Consumer rule) { for (Object prop : info().properties()) { // skip children (only properties are interesting) - if (prop != children && children.contains(prop) == false && typeToken.isInstance(prop)) { + if (prop != children && typeToken.isInstance(prop) && children.contains(prop) == false) { rule.accept((E) prop); } } @@ -202,20 +202,21 @@ public T transformUp(Class typeToken, Function protected > T transformChildren(Function traversalOperation) { boolean childrenChanged = false; - // stream() could be used but the code is just as complicated without any advantages - // further more, it would include bring in all the associated stream/collector object creation even though in - // most cases the immediate tree would be quite small (0,1,2 elements) - List transformedChildren = new ArrayList<>(children().size()); + // Avoid creating a new array of children if no change is needed. + // And when it happens, look at using replacement to minimize the amount of method invocations. + List transformedChildren = null; - for (T child : children) { + for (int i = 0, s = children.size(); i < s; i++) { + T child = children.get(i); T next = traversalOperation.apply(child); - if (child.equals(next)) { - // use the initial value - next = child; - } else { - childrenChanged = true; + if (child.equals(next) == false) { + // lazy copy + replacement in place + if (childrenChanged == false) { + childrenChanged = true; + transformedChildren = new ArrayList<>(children); + } + transformedChildren.set(i, next); } - transformedChildren.add(next); } return (childrenChanged ? replaceChildrenSameSize(transformedChildren) : (T) this); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeInfo.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeInfo.java index a9782554eadbd..b355981fe957f 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeInfo.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeInfo.java @@ -52,7 +52,7 @@ final T transform(Function rule, Class typeToken) List children = node.children(); Function realRule = p -> { - if (p != children && false == children.contains(p) && (p == null || typeToken.isInstance(p))) { + if (p != children && (p == null || typeToken.isInstance(p)) && false == children.contains(p)) { return rule.apply(typeToken.cast(p)); } return p; diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index a595eedaf4b8d..2686ab9c0b016 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -89,7 +89,7 @@ public static class Builder extends FieldMapper.Builder { } return XContentMapValues.nodeIntegerValue(o); - }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null) + }, m -> toType(m).fieldType().dims, XContentBuilder::field, Objects::toString).setSerializerCheck((id, ic, v) -> v != null) .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current)) .addValidator(dims -> { if (dims == null) { diff --git a/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java b/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java index 187126fb31e3e..c378b822ce0b0 100644 --- a/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java +++ b/x-pack/plugin/redact/src/main/java/org/elasticsearch/xpack/redact/RedactProcessor.java @@ -294,9 +294,13 @@ public void extract(byte[] utf8Bytes, int offset, Region region) { */ String redactMatches(byte[] utf8Bytes, String redactStartToken, String redactEndToken) { var merged = mergeOverlappingReplacements(replacementPositions); - int longestPatternName = merged.stream().mapToInt(r -> r.patternName.getBytes(StandardCharsets.UTF_8).length).max().getAsInt(); + int maxPatternNameLength = merged.stream() + .mapToInt(r -> r.patternName.getBytes(StandardCharsets.UTF_8).length) + .max() + .getAsInt(); - int maxPossibleLength = longestPatternName * merged.size() + utf8Bytes.length; + int maxPossibleLength = (redactStartToken.length() + maxPatternNameLength + redactEndToken.length()) * merged.size() + + utf8Bytes.length; byte[] redact = new byte[maxPossibleLength]; int readOffset = 0; diff --git a/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java b/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java index 76bf99d170a8f..bf287735d9fc3 100644 --- a/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java +++ b/x-pack/plugin/redact/src/test/java/org/elasticsearch/xpack/redact/RedactProcessorTests.java @@ -108,6 +108,18 @@ public void testMatchRedact() throws Exception { var redacted = RedactProcessor.matchRedact(input, List.of(grok)); assertEquals(" ", redacted); } + { + var config = new HashMap(); + config.put("field", "to_redact"); + config.put("patterns", List.of("%{NUMBER:NUMBER}")); + config.put("pattern_definitions", Map.of("NUMBER", "\\d{4}")); + var processor = new RedactProcessor.Factory(mockLicenseState(), MatcherWatchdog.noop()).create(null, "t", "d", config); + var grok = processor.getGroks().get(0); + + String input = "1001"; + var redacted = RedactProcessor.matchRedact(input, List.of(grok), "_prefix_", "_suffix_"); + assertEquals("_prefix_NUMBER_suffix_", redacted); + } } public void testMatchRedactMultipleGroks() throws Exception { diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java index c50fe50db8b40..c9a1a82b34118 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java @@ -90,6 +90,7 @@ public void testSearcherId() throws Exception { for (String allocatedNode : allocatedNodes) { if (randomBoolean()) { internalCluster().restartNode(allocatedNode); + ensureGreen(indexName); } } ensureGreen(indexName); @@ -151,6 +152,7 @@ public void testRetryPointInTime() throws Exception { final Set allocatedNodes = internalCluster().nodesInclude(indexName); for (String allocatedNode : allocatedNodes) { internalCluster().restartNode(allocatedNode); + ensureGreen(indexName); } ensureGreen(indexName); assertNoFailuresAndResponse( diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 84a632a419ead..86120196492c9 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -820,7 +820,7 @@ public void testSnapshotOfSearchableSnapshotIncludesNoDataButCanBeRestored() thr final String tmpRepositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createRepositoryNoVerify(tmpRepositoryName, "fs"); final Path repoPath = internalCluster().getCurrentMasterNodeInstance(Environment.class) - .resolveRepoFile( + .resolveRepoDir( clusterAdmin().prepareGetRepositories(TEST_REQUEST_TIMEOUT, tmpRepositoryName) .get() .repositories() diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSearchIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSearchIntegTests.java index c60ebc884dbed..70efccbbb29af 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSearchIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSearchIntegTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.searchablesnapshots; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchType; @@ -125,5 +126,9 @@ public void testKeywordSortedQueryOnFrozen() throws Exception { assertThat(searchResponse.getTotalShards(), equalTo(20)); assertThat(searchResponse.getHits().getTotalHits().value, equalTo(4L)); }); + + // check that field_caps empty field filtering works as well + FieldCapabilitiesResponse response = client().prepareFieldCaps(mountedIndices).setFields("*").setincludeEmptyFields(false).get(); + assertNotNull(response.getField("keyword")); } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java index b124783f90c76..26ca3d821b3a9 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java @@ -339,7 +339,7 @@ public void testCleanUpMigratedSystemIndexAfterIndicesAreDeleted() throws Except /** * Mimics migration of the {@link SearchableSnapshots#SNAPSHOT_BLOB_CACHE_INDEX} as done in - * {@link org.elasticsearch.upgrades.SystemIndexMigrator}, where the index is re-indexed, and replaced by an alias. + * org.elasticsearch.upgrades.SystemIndexMigrator, where the index is re-indexed, and replaced by an alias. */ private void migrateTheSystemIndex() { final var migratedSnapshotBlobCache = SNAPSHOT_BLOB_CACHE_INDEX + SystemIndices.UPGRADED_INDEX_SUFFIX; diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java index c955457b78d60..3534988b25ce7 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java @@ -145,7 +145,7 @@ public void testConcurrentPrewarming() throws Exception { docsPerIndex.put(indexName, nbDocs); } - final Path repositoryPath = node().getEnvironment().resolveRepoFile(randomAlphaOfLength(10)); + final Path repositoryPath = node().getEnvironment().resolveRepoDir(randomAlphaOfLength(10)); final Settings.Builder repositorySettings = Settings.builder().put("location", repositoryPath); if (randomBoolean()) { repositorySettings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES); diff --git a/x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..69eead6707114 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,8 @@ +org.elasticsearch.searchablesnapshots: + - files: + - relative_path: snapshot_cache + relative_to: data + mode: read_write + - relative_path: indices + relative_to: data + mode: read_write diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java index 53ea908ad8801..3d4d7f768c1b3 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java @@ -98,7 +98,7 @@ public void testRandomReads() throws IOException { .put("path.home", createTempDir()) .build(); final Environment environment = TestEnvironment.newEnvironment(settings); - for (Path path : environment.dataFiles()) { + for (Path path : environment.dataDirs()) { Files.createDirectories(path); } SnapshotId snapshotId = new SnapshotId("_name", "_uuid"); diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index d3425c1e42e5b..4f857655993aa 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -79,21 +79,21 @@ dependencies { runtimeOnly "joda-time:joda-time:2.10.10" // Dependencies for oidc - api "com.nimbusds:oauth2-oidc-sdk:11.10.1" + api "com.nimbusds:oauth2-oidc-sdk:11.22.2" api project(path: xpackModule('security:lib:nimbus-jose-jwt-modified'), configuration: 'shadow') if (isEclipse) { /* * Eclipse can't pick up the shadow dependency so we point it at the unmodified version of the library * so it can compile things. */ - api "com.nimbusds:nimbus-jose-jwt:9.37.3" + api "com.nimbusds:nimbus-jose-jwt:10.0.2" } - api "com.nimbusds:lang-tag:1.4.4" + api "com.nimbusds:lang-tag:1.7" api "com.sun.mail:jakarta.mail:1.6.3" api "net.jcip:jcip-annotations:1.0" - api "net.minidev:json-smart:2.5.1" - api "net.minidev:accessors-smart:2.4.2" - api "org.ow2.asm:asm:8.0.1" + api "net.minidev:json-smart:2.5.2" + api "net.minidev:accessors-smart:2.5.2" + api "org.ow2.asm:asm:9.7.1" testImplementation "org.elasticsearch:mocksocket:${versions.mocksocket}" diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java index 3994fb50c7fc6..689134a5eba17 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java @@ -102,6 +102,9 @@ import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.xpack.core.security.CommandLineHttpClient.createURL; +import static org.elasticsearch.xpack.security.cli.CertGenUtils.buildKeyUsage; +import static org.elasticsearch.xpack.security.cli.HttpCertificateCommand.DEFAULT_CA_KEY_USAGE; +import static org.elasticsearch.xpack.security.cli.HttpCertificateCommand.DEFAULT_CERT_KEY_USAGE; /** * Configures a new cluster node, by appending to the elasticsearch.yml, so that it forms a single node cluster with @@ -163,7 +166,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce final boolean inEnrollmentMode = options.has(enrollmentTokenParam); // skipping security auto-configuration because node considered as restarting. - for (Path dataPath : env.dataFiles()) { + for (Path dataPath : env.dataDirs()) { if (Files.isDirectory(dataPath) && false == isDirEmpty(dataPath)) { final String msg = "Skipping security auto configuration because it appears that the node is not starting up for the " + "first time. The node might already be part of a cluster and this auto setup utility is designed to configure " @@ -173,7 +176,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce } // pre-flight checks for the files that are going to be changed - final Path ymlPath = env.configFile().resolve("elasticsearch.yml"); + final Path ymlPath = env.configDir().resolve("elasticsearch.yml"); // it is odd for the `elasticsearch.yml` file to be missing or not be a regular (the node won't start) // but auto configuration should not be concerned with fixing it (by creating the file) and let the node startup fail if (false == Files.exists(ymlPath) || false == Files.isRegularFile(ymlPath, LinkOption.NOFOLLOW_LINKS)) { @@ -194,7 +197,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce ); notifyOfFailure(inEnrollmentMode, terminal, Terminal.Verbosity.NORMAL, ExitCodes.NOOP, msg); } - final Path keystorePath = KeyStoreWrapper.keystorePath(env.configFile()); + final Path keystorePath = KeyStoreWrapper.keystorePath(env.configDir()); // Inform that auto-configuration will not run if keystore cannot be read. if (Files.exists(keystorePath) && (false == Files.isRegularFile(keystorePath, LinkOption.NOFOLLOW_LINKS) || false == Files.isReadable(keystorePath))) { @@ -218,7 +221,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce checkExistingConfiguration(env.settings(), inEnrollmentMode, terminal); final ZonedDateTime autoConfigDate = ZonedDateTime.now(ZoneOffset.UTC); - final Path tempGeneratedTlsCertsDir = env.configFile() + final Path tempGeneratedTlsCertsDir = env.configDir() .resolve(String.format(Locale.ROOT, TLS_GENERATED_CERTS_DIR_NAME + ".%d.tmp", autoConfigDate.toInstant().getEpochSecond())); try { // it is useful to pre-create the sub-config dir in order to check that the config dir is writable and that file owners match @@ -247,12 +250,12 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce // If the node process works OK given the owner of the config dir, it should also tolerate the auto-created config dir, // provided that they both have the same owner and permissions. final UserPrincipal newFileOwner = Files.getOwner(tempGeneratedTlsCertsDir, LinkOption.NOFOLLOW_LINKS); - if (false == newFileOwner.equals(Files.getOwner(env.configFile(), LinkOption.NOFOLLOW_LINKS))) { + if (false == newFileOwner.equals(Files.getOwner(env.configDir(), LinkOption.NOFOLLOW_LINKS))) { // the following is only printed once, if the node starts successfully UserException userException = new UserException( ExitCodes.CONFIG, "Aborting auto configuration because of config dir ownership mismatch. Config dir is owned by " - + Files.getOwner(env.configFile(), LinkOption.NOFOLLOW_LINKS).getName() + + Files.getOwner(env.configDir(), LinkOption.NOFOLLOW_LINKS).getName() + " but auto-configuration directory would be owned by " + newFileOwner.getName() ); @@ -411,7 +414,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce null, true, TRANSPORT_CA_CERTIFICATE_DAYS, - SIGNATURE_ALGORITHM + SIGNATURE_ALGORITHM, + null, + Set.of() ); // transport key/certificate final KeyPair transportKeyPair = CertGenUtils.generateKeyPair(TRANSPORT_KEY_SIZE); @@ -424,7 +429,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce transportCaKey, false, TRANSPORT_CERTIFICATE_DAYS, - SIGNATURE_ALGORITHM + SIGNATURE_ALGORITHM, + null, + Set.of() ); final KeyPair httpCaKeyPair = CertGenUtils.generateKeyPair(HTTP_CA_KEY_SIZE); @@ -438,7 +445,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce null, true, HTTP_CA_CERTIFICATE_DAYS, - SIGNATURE_ALGORITHM + SIGNATURE_ALGORITHM, + buildKeyUsage(DEFAULT_CA_KEY_USAGE), + Set.of() ); } catch (Throwable t) { try { @@ -464,6 +473,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce false, HTTP_CERTIFICATE_DAYS, SIGNATURE_ALGORITHM, + buildKeyUsage(DEFAULT_CERT_KEY_USAGE), Set.of(new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)) ); @@ -496,7 +506,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce } // save the existing keystore before replacing - final Path keystoreBackupPath = env.configFile() + final Path keystoreBackupPath = env.configDir() .resolve( String.format(Locale.ROOT, KeyStoreWrapper.KEYSTORE_FILENAME + ".%d.orig", autoConfigDate.toInstant().getEpochSecond()) ); @@ -514,7 +524,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce } final SetOnce nodeKeystorePassword = new SetOnce<>(); - try (KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configFile(), () -> { + try (KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configDir(), () -> { nodeKeystorePassword.set(new SecureString(terminal.readSecret(""))); return nodeKeystorePassword.get().clone(); })) { @@ -581,7 +591,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce nodeKeystore.setString("xpack.security.http.ssl.keystore.secure_password", httpKeystorePassword.getChars()); } // finally overwrites the node keystore (if the keystores have been successfully written) - nodeKeystore.save(env.configFile(), nodeKeystorePassword.get() == null ? new char[0] : nodeKeystorePassword.get().getChars()); + nodeKeystore.save(env.configDir(), nodeKeystorePassword.get() == null ? new char[0] : nodeKeystorePassword.get().getChars()); } catch (Throwable t) { // restore keystore to revert possible keystore bootstrap try { @@ -614,10 +624,10 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce try { // all certs and keys have been generated in the temp certs dir, therefore: // 1. backup (move) any previously existing tls certs dir (this backup is NOT removed when auto-conf finishes) - if (Files.exists(env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME))) { + if (Files.exists(env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME))) { moveDirectory( - env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME), - env.configFile() + env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME), + env.configDir() .resolve( String.format( Locale.ROOT, @@ -628,7 +638,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce ); } // 2. move the newly populated temp certs dir to its permanent static dir name - moveDirectory(tempGeneratedTlsCertsDir, env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME)); + moveDirectory(tempGeneratedTlsCertsDir, env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME)); } catch (Throwable t) { // restore keystore to revert possible keystore bootstrap try { @@ -649,7 +659,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce // revert any previously existing TLS certs try { if (Files.exists( - env.configFile() + env.configDir() .resolve( String.format( Locale.ROOT, @@ -659,7 +669,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce ) )) { moveDirectory( - env.configFile() + env.configDir() .resolve( String.format( Locale.ROOT, @@ -667,7 +677,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce autoConfigDate.toInstant().getEpochSecond() ) ), - env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME) + env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME) ); } } catch (Exception ex) { @@ -686,7 +696,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce final Environment localFinalEnv = env; final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss", Locale.ROOT); List existingConfigLines = Files.readAllLines(ymlPath, StandardCharsets.UTF_8); - fullyWriteFile(env.configFile(), "elasticsearch.yml", true, stream -> { + fullyWriteFile(env.configDir(), "elasticsearch.yml", true, stream -> { try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8))) { // start with the existing config lines for (String line : existingConfigLines) { @@ -827,16 +837,16 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce } try { // this removes a statically named directory, so it is potentially dangerous - deleteDirectory(env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME)); + deleteDirectory(env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME)); } catch (Exception ex) { t.addSuppressed(ex); } - Path backupCertsDir = env.configFile() + Path backupCertsDir = env.configDir() .resolve( String.format(Locale.ROOT, TLS_GENERATED_CERTS_DIR_NAME + ".%d.orig", autoConfigDate.toInstant().getEpochSecond()) ); if (Files.exists(backupCertsDir)) { - moveDirectory(backupCertsDir, env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME)); + moveDirectory(backupCertsDir, env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME)); } throw t; } @@ -887,14 +897,14 @@ private Environment possiblyReconfigureNode(Environment env, Terminal terminal, // with --enrolment-token token, in the first place. final List existingConfigLines; try { - existingConfigLines = Files.readAllLines(env.configFile().resolve("elasticsearch.yml"), StandardCharsets.UTF_8); + existingConfigLines = Files.readAllLines(env.configDir().resolve("elasticsearch.yml"), StandardCharsets.UTF_8); } catch (IOException e) { // This shouldn't happen, we would have failed earlier but we need to catch the exception throw new UserException(ExitCodes.IO_ERROR, "Aborting enrolling to cluster. Unable to read elasticsearch.yml.", e); } final List existingConfigWithoutAutoconfiguration = removePreviousAutoconfiguration(existingConfigLines); if (false == existingConfigLines.equals(existingConfigWithoutAutoconfiguration) - && Files.exists(env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME))) { + && Files.exists(env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME))) { terminal.println(""); terminal.println("This node will be reconfigured to join an existing cluster, using the enrollment token that you provided."); terminal.println("This operation will overwrite the existing configuration. Specifically: "); @@ -907,7 +917,7 @@ private Environment possiblyReconfigureNode(Environment env, Terminal terminal, } removeAutoConfigurationFromKeystore(env, terminal); try { - fullyWriteFile(env.configFile(), "elasticsearch.yml", true, stream -> { + fullyWriteFile(env.configDir(), "elasticsearch.yml", true, stream -> { try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(stream, StandardCharsets.UTF_8))) { for (String l : existingConfigWithoutAutoconfiguration) { bw.write(l); @@ -915,7 +925,7 @@ private Environment possiblyReconfigureNode(Environment env, Terminal terminal, } } }); - deleteDirectory(env.configFile().resolve(TLS_GENERATED_CERTS_DIR_NAME)); + deleteDirectory(env.configDir().resolve(TLS_GENERATED_CERTS_DIR_NAME)); } catch (Throwable t) { throw new UserException( ExitCodes.IO_ERROR, @@ -1262,9 +1272,9 @@ static List removePreviousAutoconfiguration(List existingConfigL } private static void removeAutoConfigurationFromKeystore(Environment env, Terminal terminal) throws UserException { - if (Files.exists(KeyStoreWrapper.keystorePath(env.configFile()))) { + if (Files.exists(KeyStoreWrapper.keystorePath(env.configDir()))) { try ( - KeyStoreWrapper existingKeystore = KeyStoreWrapper.load(env.configFile()); + KeyStoreWrapper existingKeystore = KeyStoreWrapper.load(env.configDir()); SecureString keystorePassword = existingKeystore.hasPassword() ? new SecureString(terminal.readSecret("Enter password for the elasticsearch keystore: ")) : new SecureString(new char[0]); @@ -1288,7 +1298,7 @@ private static void removeAutoConfigurationFromKeystore(Environment env, Termina } existingKeystore.remove(setting); } - existingKeystore.save(env.configFile(), keystorePassword.getChars()); + existingKeystore.save(env.configDir(), keystorePassword.getChars()); } catch (Exception e) { terminal.errorPrintln(Terminal.Verbosity.VERBOSE, ""); terminal.errorPrintln(Terminal.Verbosity.VERBOSE, ExceptionsHelper.stackTrace(e)); diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java index c3f4d8a57b560..b82e2e1d77faf 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java @@ -20,6 +20,7 @@ import org.bouncycastle.asn1.x509.ExtensionsGenerator; import org.bouncycastle.asn1.x509.GeneralName; import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; import org.bouncycastle.asn1.x509.Time; import org.bouncycastle.cert.CertIOException; import org.bouncycastle.cert.X509CertificateHolder; @@ -53,10 +54,14 @@ import java.sql.Date; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import javax.net.ssl.X509ExtendedKeyManager; import javax.net.ssl.X509ExtendedTrustManager; @@ -73,14 +78,33 @@ public class CertGenUtils { private static final int SERIAL_BIT_LENGTH = 20 * 8; private static final BouncyCastleProvider BC_PROV = new BouncyCastleProvider(); + /** + * The mapping of key usage names to their corresponding integer values as defined in {@code KeyUsage} class. + */ + public static final Map KEY_USAGE_MAPPINGS = Collections.unmodifiableMap( + new TreeMap<>( + Map.ofEntries( + Map.entry("digitalSignature", KeyUsage.digitalSignature), + Map.entry("nonRepudiation", KeyUsage.nonRepudiation), + Map.entry("keyEncipherment", KeyUsage.keyEncipherment), + Map.entry("dataEncipherment", KeyUsage.dataEncipherment), + Map.entry("keyAgreement", KeyUsage.keyAgreement), + Map.entry("keyCertSign", KeyUsage.keyCertSign), + Map.entry("cRLSign", KeyUsage.cRLSign), + Map.entry("encipherOnly", KeyUsage.encipherOnly), + Map.entry("decipherOnly", KeyUsage.decipherOnly) + ) + ) + ); + private CertGenUtils() {} /** * Generates a CA certificate */ - public static X509Certificate generateCACertificate(X500Principal x500Principal, KeyPair keyPair, int days) + public static X509Certificate generateCACertificate(X500Principal x500Principal, KeyPair keyPair, int days, KeyUsage keyUsage) throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException { - return generateSignedCertificate(x500Principal, null, keyPair, null, null, true, days, null); + return generateSignedCertificate(x500Principal, null, keyPair, null, null, true, days, null, keyUsage, Set.of()); } /** @@ -107,7 +131,7 @@ public static X509Certificate generateSignedCertificate( PrivateKey caPrivKey, int days ) throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException { - return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, null); + return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, null, null, Set.of()); } /** @@ -123,54 +147,14 @@ public static X509Certificate generateSignedCertificate( * certificate * @param caPrivKey the CA private key. If {@code null}, this results in a self signed * certificate - * @param days no of days certificate will be valid from now - * @param signatureAlgorithm algorithm used for signing certificate. If {@code null} or - * empty, then use default algorithm {@link CertGenUtils#getDefaultSignatureAlgorithm(PrivateKey)} - * @return a signed {@link X509Certificate} - */ - public static X509Certificate generateSignedCertificate( - X500Principal principal, - GeneralNames subjectAltNames, - KeyPair keyPair, - X509Certificate caCert, - PrivateKey caPrivKey, - int days, - String signatureAlgorithm - ) throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException { - return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, signatureAlgorithm); - } - - /** - * Generates a signed certificate - * - * @param principal the principal of the certificate; commonly referred to as the - * distinguished name (DN) - * @param subjectAltNames the subject alternative names that should be added to the - * certificate as an X509v3 extension. May be {@code null} - * @param keyPair the key pair that will be associated with the certificate - * @param caCert the CA certificate. If {@code null}, this results in a self signed - * certificate - * @param caPrivKey the CA private key. If {@code null}, this results in a self signed - * certificate * @param isCa whether or not the generated certificate is a CA * @param days no of days certificate will be valid from now * @param signatureAlgorithm algorithm used for signing certificate. If {@code null} or * empty, then use default algorithm {@link CertGenUtils#getDefaultSignatureAlgorithm(PrivateKey)} + * @param keyUsage the key usage that should be added to the certificate as a X509v3 extension (can be {@code null}) + * @param extendedKeyUsages the extended key usages that should be added to the certificate as a X509v3 extension (can be empty) * @return a signed {@link X509Certificate} */ - public static X509Certificate generateSignedCertificate( - X500Principal principal, - GeneralNames subjectAltNames, - KeyPair keyPair, - X509Certificate caCert, - PrivateKey caPrivKey, - boolean isCa, - int days, - String signatureAlgorithm - ) throws NoSuchAlgorithmException, CertificateException, CertIOException, OperatorCreationException { - return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, isCa, days, signatureAlgorithm, Set.of()); - } - public static X509Certificate generateSignedCertificate( X500Principal principal, GeneralNames subjectAltNames, @@ -180,6 +164,7 @@ public static X509Certificate generateSignedCertificate( boolean isCa, int days, String signatureAlgorithm, + KeyUsage keyUsage, Set extendedKeyUsages ) throws NoSuchAlgorithmException, CertificateException, CertIOException, OperatorCreationException { Objects.requireNonNull(keyPair, "Key-Pair must not be null"); @@ -198,6 +183,7 @@ public static X509Certificate generateSignedCertificate( notBefore, notAfter, signatureAlgorithm, + keyUsage, extendedKeyUsages ); } @@ -223,6 +209,7 @@ public static X509Certificate generateSignedCertificate( notBefore, notAfter, signatureAlgorithm, + null, Set.of() ); } @@ -237,6 +224,7 @@ public static X509Certificate generateSignedCertificate( ZonedDateTime notBefore, ZonedDateTime notAfter, String signatureAlgorithm, + KeyUsage keyUsage, Set extendedKeyUsages ) throws NoSuchAlgorithmException, CertIOException, OperatorCreationException, CertificateException { final BigInteger serial = CertGenUtils.getSerial(); @@ -272,6 +260,11 @@ public static X509Certificate generateSignedCertificate( } builder.addExtension(Extension.basicConstraints, isCa, new BasicConstraints(isCa)); + if (keyUsage != null) { + // as per RFC 5280 (section 4.2.1.3), if the key usage is present, then it SHOULD be marked as critical. + final boolean isCritical = true; + builder.addExtension(Extension.keyUsage, isCritical, keyUsage); + } if (extendedKeyUsages != null) { for (ExtendedKeyUsage extendedKeyUsage : extendedKeyUsages) { builder.addExtension(Extension.extendedKeyUsage, false, extendedKeyUsage); @@ -318,7 +311,7 @@ private static String getDefaultSignatureAlgorithm(PrivateKey key) { */ static PKCS10CertificationRequest generateCSR(KeyPair keyPair, X500Principal principal, GeneralNames sanList) throws IOException, OperatorCreationException { - return generateCSR(keyPair, principal, sanList, Set.of()); + return generateCSR(keyPair, principal, sanList, null, Set.of()); } /** @@ -335,6 +328,7 @@ static PKCS10CertificationRequest generateCSR( KeyPair keyPair, X500Principal principal, GeneralNames sanList, + KeyUsage keyUsage, Set extendedKeyUsages ) throws IOException, OperatorCreationException { Objects.requireNonNull(keyPair, "Key-Pair must not be null"); @@ -347,7 +341,9 @@ static PKCS10CertificationRequest generateCSR( if (sanList != null) { extGen.addExtension(Extension.subjectAlternativeName, false, sanList); } - + if (keyUsage != null) { + extGen.addExtension(Extension.keyUsage, true, keyUsage); + } for (ExtendedKeyUsage extendedKeyUsage : extendedKeyUsages) { extGen.addExtension(Extension.extendedKeyUsage, false, extendedKeyUsage); } @@ -430,4 +426,31 @@ public static GeneralName createCommonName(String cn) { public static String buildDnFromDomain(String domain) { return "DC=" + domain.replace(".", ",DC="); } + + public static KeyUsage buildKeyUsage(Collection keyUsages) { + if (keyUsages == null || keyUsages.isEmpty()) { + return null; + } + + int usageBits = 0; + for (String keyUsageName : keyUsages) { + Integer keyUsageValue = findKeyUsageByName(keyUsageName); + if (keyUsageValue == null) { + throw new IllegalArgumentException("Unknown keyUsage: " + keyUsageName); + } + usageBits |= keyUsageValue; + } + return new KeyUsage(usageBits); + } + + public static boolean isValidKeyUsage(String keyUsage) { + return findKeyUsageByName(keyUsage) != null; + } + + private static Integer findKeyUsageByName(String keyUsageName) { + if (keyUsageName == null) { + return null; + } + return KEY_USAGE_MAPPINGS.get(keyUsageName.trim()); + } } diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java index a6716f0360a1b..a342e3cca3e94 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java @@ -403,7 +403,7 @@ static CAInfo getCAInfo( // generate the CA keys and cert X500Principal x500Principal = new X500Principal(dn); KeyPair keyPair = CertGenUtils.generateKeyPair(keysize); - Certificate caCert = CertGenUtils.generateCACertificate(x500Principal, keyPair, days); + Certificate caCert = CertGenUtils.generateCACertificate(x500Principal, keyPair, days, null); final char[] password; if (prompt) { password = terminal.readSecret("Enter password for CA private key: "); diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java index a9c0653716851..b64e21786279b 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java @@ -15,6 +15,7 @@ import org.bouncycastle.asn1.DERIA5String; import org.bouncycastle.asn1.x509.GeneralName; import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openssl.PEMEncryptor; import org.bouncycastle.openssl.jcajce.JcaPEMWriter; @@ -110,6 +111,7 @@ class CertificateTool extends MultiCommand { "[a-zA-Z0-9!@#$%^&{}\\[\\]()_+\\-=,.~'` ]{1," + MAX_FILENAME_LENGTH + "}" ); private static final int DEFAULT_KEY_SIZE = 2048; + static final List DEFAULT_CA_KEY_USAGE = List.of("keyCertSign", "cRLSign"); // Older versions of OpenSSL had a max internal password length. // We issue warnings when writing files with passwords that would not be usable in those versions of OpenSSL. @@ -202,6 +204,7 @@ abstract static class CertificateCommand extends EnvironmentAwareCommand { final OptionSpec outputPathSpec; final OptionSpec outputPasswordSpec; final OptionSpec keysizeSpec; + OptionSpec caKeyUsageSpec; OptionSpec pemFormatSpec; OptionSpec daysSpec; @@ -274,6 +277,16 @@ final void acceptInputFile() { inputFileSpec = parser.accepts("in", "file containing details of the instances in yaml format").withRequiredArg(); } + final void acceptCertificateAuthorityKeyUsage() { + caKeyUsageSpec = parser.accepts( + "keyusage", + "comma separated key usages to use for the generated CA. " + + "defaults to '" + + Strings.collectionToCommaDelimitedString(DEFAULT_CA_KEY_USAGE) + + "'" + ).withRequiredArg(); + } + // For testing OptionParser getParser() { return parser; @@ -309,6 +322,23 @@ final int getKeySize(OptionSet options) { } } + final List getCaKeyUsage(OptionSet options) { + if (options.has(caKeyUsageSpec)) { + final Function> splitByComma = v -> Stream.of(Strings.splitStringByCommaToArray(v)); + final List caKeyUsage = caKeyUsageSpec.values(options) + .stream() + .flatMap(splitByComma) + .filter(v -> false == Strings.isNullOrEmpty(v)) + .toList(); + if (caKeyUsage.isEmpty()) { + return DEFAULT_CA_KEY_USAGE; + } + return caKeyUsage; + } else { + return DEFAULT_CA_KEY_USAGE; + } + } + final int getDays(OptionSet options) { if (options.has(daysSpec)) { return daysSpec.value(options); @@ -396,7 +426,8 @@ CAInfo generateCA(Terminal terminal, OptionSet options) throws Exception { } X500Principal x500Principal = new X500Principal(dn); KeyPair keyPair = CertGenUtils.generateKeyPair(getKeySize(options)); - X509Certificate caCert = CertGenUtils.generateCACertificate(x500Principal, keyPair, getDays(options)); + final KeyUsage caKeyUsage = CertGenUtils.buildKeyUsage(getCaKeyUsage(options)); + X509Certificate caCert = CertGenUtils.generateCACertificate(x500Principal, keyPair, getDays(options), caKeyUsage); if (options.hasArgument(caPasswordSpec)) { char[] password = getChars(caPasswordSpec.value(options)); @@ -933,9 +964,7 @@ private static CertificateAndKey generateCertificateAndKey( keyPair, null, null, - false, - days, - null + days ); } return new CertificateAndKey((X509Certificate) certificate, keyPair.getPrivate()); @@ -949,6 +978,7 @@ static class CertificateAuthorityCommand extends CertificateCommand { super("generate a new local certificate authority"); acceptCertificateGenerationOptions(); acceptsCertificateAuthorityName(); + acceptCertificateAuthorityKeyUsage(); super.caPasswordSpec = super.outputPasswordSpec; } diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommand.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommand.java index b67bb9898991f..23a7bb96be2af 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommand.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommand.java @@ -68,6 +68,8 @@ import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -80,7 +82,9 @@ import javax.security.auth.x500.X500Principal; +import static org.elasticsearch.xpack.security.cli.CertGenUtils.buildKeyUsage; import static org.elasticsearch.xpack.security.cli.CertGenUtils.generateSignedCertificate; +import static org.elasticsearch.xpack.security.cli.CertGenUtils.isValidKeyUsage; /** * This command is the "elasticsearch-certutil http" command. It provides a guided process for creating @@ -95,7 +99,8 @@ class HttpCertificateCommand extends EnvironmentAwareCommand { static final X500Principal DEFAULT_CA_NAME = new X500Principal("CN=Elasticsearch HTTP CA"); static final int DEFAULT_CA_KEY_SIZE = DEFAULT_CERT_KEY_SIZE; static final Period DEFAULT_CA_VALIDITY = DEFAULT_CERT_VALIDITY; - + static final List DEFAULT_CA_KEY_USAGE = List.of("keyCertSign", "cRLSign"); + static final List DEFAULT_CERT_KEY_USAGE = List.of("digitalSignature", "keyEncipherment"); private static final String ES_README_CSR = "es-readme-csr.txt"; private static final String ES_YML_CSR = "es-sample-csr.yml"; private static final String ES_README_P12 = "es-readme-p12.txt"; @@ -133,14 +138,24 @@ private class CertOptions { final List dnsNames; final List ipNames; final int keySize; + final List keyUsage; final Period validity; - private CertOptions(String name, X500Principal subject, List dnsNames, List ipNames, int keySize, Period validity) { + private CertOptions( + String name, + X500Principal subject, + List dnsNames, + List ipNames, + int keySize, + List keyUsage, + Period validity + ) { this.name = name; this.subject = subject; this.dnsNames = dnsNames; this.ipNames = ipNames; this.keySize = keySize; + this.keyUsage = keyUsage; this.validity = validity; } } @@ -194,6 +209,7 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce terminal.println(Terminal.Verbosity.VERBOSE, "\tDNS Names: " + Strings.collectionToCommaDelimitedString(cert.dnsNames)); terminal.println(Terminal.Verbosity.VERBOSE, "\tIP Names: " + Strings.collectionToCommaDelimitedString(cert.ipNames)); terminal.println(Terminal.Verbosity.VERBOSE, "\tKey Size: " + cert.keySize); + terminal.println(Terminal.Verbosity.VERBOSE, "\tKey Usage: " + Strings.collectionToCommaDelimitedString(cert.keyUsage)); terminal.println(Terminal.Verbosity.VERBOSE, "\tValidity: " + toString(cert.validity)); certificates.add(cert); @@ -339,6 +355,7 @@ private void writeCertificateAndKeyDetails( keyPair, cert.subject, sanList, + buildKeyUsage(cert.keyUsage), Set.of(new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)) ); final String csrFile = "http-" + cert.name + ".csr"; @@ -372,6 +389,7 @@ private void writeCertificateAndKeyDetails( notBefore, notAfter, null, + buildKeyUsage(cert.keyUsage), Set.of(new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)) ); @@ -508,7 +526,7 @@ private static Map buildSubstitutions(Environment env, Map keyUsage = DEFAULT_CERT_KEY_USAGE; while (true) { terminal.println(Terminal.Verbosity.SILENT, "Key Name: " + certName); terminal.println(Terminal.Verbosity.SILENT, "Subject DN: " + dn); terminal.println(Terminal.Verbosity.SILENT, "Key Size: " + keySize); + terminal.println(Terminal.Verbosity.SILENT, "Key Usage: " + Strings.collectionToCommaDelimitedString(keyUsage)); terminal.println(Terminal.Verbosity.SILENT, ""); if (terminal.promptYesNo("Do you wish to change any of these options?", false) == false) { break; @@ -736,9 +756,22 @@ private CertOptions getCertificateConfiguration( keySize = readKeySize(terminal, keySize); terminal.println(""); + + printHeader("What key usage should your certificate have?", terminal); + terminal.println("The key usage extension defines the purpose of the key contained in the certificate."); + terminal.println("The usage restriction might be employed when a key, that could be used for more than "); + terminal.println("one operation, is to be restricted."); + terminal.println("You may enter the key usage as a comma-delimited list of following values: "); + for (String keyUsageName : CertGenUtils.KEY_USAGE_MAPPINGS.keySet()) { + terminal.println(" - " + keyUsageName); + } + terminal.println(""); + + keyUsage = readKeyUsage(terminal, keyUsage); + terminal.println(""); } - return new CertOptions(certName, dn, dnsNames, ipNames, keySize, validity); + return new CertOptions(certName, dn, dnsNames, ipNames, keySize, keyUsage, validity); } private static String validateHostname(String name) { @@ -859,10 +892,12 @@ private CertificateTool.CAInfo createNewCA(Terminal terminal) { X500Principal dn = DEFAULT_CA_NAME; Period validity = DEFAULT_CA_VALIDITY; int keySize = DEFAULT_CA_KEY_SIZE; + List keyUsage = DEFAULT_CA_KEY_USAGE; while (true) { terminal.println(Terminal.Verbosity.SILENT, "Subject DN: " + dn); terminal.println(Terminal.Verbosity.SILENT, "Validity: " + toString(validity)); terminal.println(Terminal.Verbosity.SILENT, "Key Size: " + keySize); + terminal.println(Terminal.Verbosity.SILENT, "Key Usage: " + Strings.collectionToCommaDelimitedString(keyUsage)); terminal.println(Terminal.Verbosity.SILENT, ""); if (terminal.promptYesNo("Do you wish to change any of these options?", false) == false) { break; @@ -904,13 +939,38 @@ private CertificateTool.CAInfo createNewCA(Terminal terminal) { keySize = readKeySize(terminal, keySize); terminal.println(""); + + printHeader("What key usage should your CA have?", terminal); + terminal.println("The key usage extension defines the purpose of the key contained in the certificate."); + terminal.println("The usage restriction might be employed when a key, that could be used for more than "); + terminal.println("one operation, is to be restricted."); + terminal.println("You may enter the key usage as a comma-delimited list of following values: "); + for (String keyUsageName : CertGenUtils.KEY_USAGE_MAPPINGS.keySet()) { + terminal.println(" - " + keyUsageName); + } + terminal.println(""); + + keyUsage = readKeyUsage(terminal, keyUsage); + terminal.println(""); } try { final KeyPair keyPair = CertGenUtils.generateKeyPair(keySize); final ZonedDateTime notBefore = ZonedDateTime.now(ZoneOffset.UTC); final ZonedDateTime notAfter = notBefore.plus(validity); - X509Certificate caCert = generateSignedCertificate(dn, null, keyPair, null, null, true, notBefore, notAfter, null); + X509Certificate caCert = generateSignedCertificate( + dn, + null, + keyPair, + null, + null, + true, + notBefore, + notAfter, + null, + buildKeyUsage(keyUsage), + Set.of() + ); printHeader("CA password", terminal); terminal.println("We recommend that you protect your CA private key with a strong password."); @@ -979,6 +1039,31 @@ private static Integer readKeySize(Terminal terminal, int keySize) { }); } + private static List readKeyUsage(Terminal terminal, List defaultKeyUsage) { + return tryReadInput(terminal, "Key Usage", defaultKeyUsage, input -> { + final String[] keyUsages = input.split(","); + final List resolvedKeyUsages = new ArrayList<>(keyUsages.length); + for (String keyUsage : keyUsages) { + keyUsage = keyUsage.trim(); + if (keyUsage.isEmpty()) { + terminal.println("Key usage cannot be blank or empty"); + return null; + } + if (isValidKeyUsage(keyUsage) == false) { + terminal.println("Invalid key usage: " + keyUsage); + terminal.println("The key usage should be one of the following values: "); + for (String keyUsageName : CertGenUtils.KEY_USAGE_MAPPINGS.keySet()) { + terminal.println(" - " + keyUsageName); + } + terminal.println(""); + return null; + } + resolvedKeyUsages.add(keyUsage); + } + return Collections.unmodifiableList(resolvedKeyUsages); + }); + } + private static char[] readPassword(Terminal terminal, String prompt, boolean confirm) { while (true) { final char[] password = terminal.readSecret(prompt + " [ for none]"); @@ -1080,7 +1165,14 @@ private static boolean askExistingCertificateAuthority(Terminal terminal) { } private static T tryReadInput(Terminal terminal, String prompt, T defaultValue, Function parser) { - final String defaultStr = defaultValue instanceof Period ? toString((Period) defaultValue) : String.valueOf(defaultValue); + final String defaultStr; + if (defaultValue instanceof Period) { + defaultStr = toString((Period) defaultValue); + } else if (defaultValue instanceof Collection collection) { + defaultStr = Strings.collectionToCommaDelimitedString(collection); + } else { + defaultStr = String.valueOf(defaultValue); + } while (true) { final String input = terminal.readText(prompt + " [" + defaultStr + "] "); if (Strings.isEmpty(input)) { @@ -1116,7 +1208,7 @@ static String toString(Period period) { private static Path requestPath(String prompt, Terminal terminal, Environment env, boolean requireExisting) { for (;;) { final String input = terminal.readText(prompt); - final Path path = env.configFile().resolve(input).toAbsolutePath(); + final Path path = env.configDir().resolve(input).toAbsolutePath(); if (path.getFileName() == null) { terminal.println(Terminal.Verbosity.SILENT, input + " is not a valid file"); diff --git a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/AutoConfigureNodeTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/AutoConfigureNodeTests.java index 129d85d0818b2..8330fb5d575ac 100644 --- a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/AutoConfigureNodeTests.java +++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/AutoConfigureNodeTests.java @@ -37,6 +37,7 @@ import static org.elasticsearch.xpack.security.cli.AutoConfigureNode.AUTO_CONFIG_TRANSPORT_ALT_DN; import static org.elasticsearch.xpack.security.cli.AutoConfigureNode.anyRemoteHostNodeAddress; import static org.elasticsearch.xpack.security.cli.AutoConfigureNode.removePreviousAutoconfiguration; +import static org.elasticsearch.xpack.security.cli.CertGenUtilsTests.assertExpectedKeyUsage; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -149,7 +150,7 @@ public void testSubjectAndIssuerForGeneratedCertificates() throws Exception { } } - public void testGeneratedHTTPCertificateSANs() throws Exception { + public void testGeneratedHTTPCertificateSANsAndKeyUsage() throws Exception { // test no publish settings Path tempDir = createTempDir(); try { @@ -180,7 +181,7 @@ public void testGeneratedHTTPCertificateSANs() throws Exception { assertThat(checkGeneralNameSan(httpCertificate, "localhost", GeneralName.dNSName), is(true)); assertThat(checkGeneralNameSan(httpCertificate, "172.168.1.100", GeneralName.iPAddress), is(true)); assertThat(checkGeneralNameSan(httpCertificate, "10.10.10.100", GeneralName.iPAddress), is(false)); - verifyExtendedKeyUsage(httpCertificate); + verifyKeyUsageAndExtendedKeyUsage(httpCertificate); } finally { deleteDirectory(tempDir); } @@ -202,7 +203,7 @@ public void testGeneratedHTTPCertificateSANs() throws Exception { assertThat(checkGeneralNameSan(httpCertificate, "localhost", GeneralName.dNSName), is(true)); assertThat(checkGeneralNameSan(httpCertificate, "172.168.1.100", GeneralName.iPAddress), is(false)); assertThat(checkGeneralNameSan(httpCertificate, "10.10.10.100", GeneralName.iPAddress), is(true)); - verifyExtendedKeyUsage(httpCertificate); + verifyKeyUsageAndExtendedKeyUsage(httpCertificate); } finally { deleteDirectory(tempDir); } @@ -228,7 +229,7 @@ public void testGeneratedHTTPCertificateSANs() throws Exception { assertThat(checkGeneralNameSan(httpCertificate, "balkan.beast", GeneralName.dNSName), is(true)); assertThat(checkGeneralNameSan(httpCertificate, "172.168.1.100", GeneralName.iPAddress), is(false)); assertThat(checkGeneralNameSan(httpCertificate, "10.10.10.100", GeneralName.iPAddress), is(false)); - verifyExtendedKeyUsage(httpCertificate); + verifyKeyUsageAndExtendedKeyUsage(httpCertificate); } finally { deleteDirectory(tempDir); } @@ -288,11 +289,12 @@ private boolean checkSubjectAndIssuerDN(X509Certificate certificate, String subj return false; } - private void verifyExtendedKeyUsage(X509Certificate httpCertificate) throws Exception { + private void verifyKeyUsageAndExtendedKeyUsage(X509Certificate httpCertificate) throws Exception { List extendedKeyUsage = httpCertificate.getExtendedKeyUsage(); assertEquals("Only one extended key usage expected for HTTP certificate.", 1, extendedKeyUsage.size()); String expectedServerAuthUsage = KeyPurposeId.id_kp_serverAuth.toASN1Primitive().toString(); assertEquals("Expected serverAuth extended key usage.", expectedServerAuthUsage, extendedKeyUsage.get(0)); + assertExpectedKeyUsage(httpCertificate, HttpCertificateCommand.DEFAULT_CERT_KEY_USAGE); } private X509Certificate runAutoConfigAndReturnHTTPCertificate(Path configDir, Settings settings) throws Exception { @@ -311,7 +313,7 @@ private Tuple runAutoConfigAndReturnCertificat SecureString httpKeystorePassword = nodeKeystore.getString("xpack.security.http.ssl.keystore.secure_password"); SecureString transportKeystorePassword = nodeKeystore.getString("xpack.security.transport.ssl.keystore.secure_password"); - final Settings newSettings = Settings.builder().loadFromPath(env.configFile().resolve("elasticsearch.yml")).build(); + final Settings newSettings = Settings.builder().loadFromPath(env.configDir().resolve("elasticsearch.yml")).build(); final String httpKeystorePath = newSettings.get("xpack.security.http.ssl.keystore.path"); final String transportKeystorePath = newSettings.get("xpack.security.transport.ssl.keystore.path"); diff --git a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java index 5c1f5a97d4335..ae905d179688b 100644 --- a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java +++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java @@ -7,10 +7,13 @@ package org.elasticsearch.xpack.security.cli; +import com.unboundid.util.ssl.cert.KeyUsageExtension; + import org.bouncycastle.asn1.x509.ExtendedKeyUsage; import org.bouncycastle.asn1.x509.GeneralName; import org.bouncycastle.asn1.x509.GeneralNames; import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.asn1.x509.KeyUsage; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.SuppressForbidden; @@ -27,22 +30,57 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedTrustManager; import javax.security.auth.x500.X500Principal; +import static org.elasticsearch.xpack.security.cli.CertGenUtils.KEY_USAGE_MAPPINGS; +import static org.elasticsearch.xpack.security.cli.CertGenUtils.buildKeyUsage; +import static org.elasticsearch.xpack.security.cli.CertGenUtils.isValidKeyUsage; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * Unit tests for cert utils */ public class CertGenUtilsTests extends ESTestCase { + /** + * The mapping of key usage names to their corresponding bit index as defined in {@code KeyUsage} class: + * + *
    + *
  • digitalSignature (0)
    • + *
  • nonRepudiation (1)
    • + *
  • keyEncipherment (2)
    • + *
  • dataEncipherment (3)
    • + *
  • keyAgreement (4)
    • + *
  • keyCertSign (5)
    • + *
  • cRLSign (6)
    • + *
  • encipherOnly (7)
    • + *
  • decipherOnly (8)
    • + *
+ */ + private static final Map KEY_USAGE_BITS = Map.ofEntries( + Map.entry("digitalSignature", 0), + Map.entry("nonRepudiation", 1), + Map.entry("keyEncipherment", 2), + Map.entry("dataEncipherment", 3), + Map.entry("keyAgreement", 4), + Map.entry("keyCertSign", 5), + Map.entry("cRLSign", 6), + Map.entry("encipherOnly", 7), + Map.entry("decipherOnly", 8) + ); + @BeforeClass public static void muteInFips() { assumeFalse("Can't run in a FIPS JVM", inFipsJvm()); @@ -103,6 +141,7 @@ public void testIssuerCertSubjectDN() throws Exception { // root CA final X500Principal rootCaPrincipal = new X500Principal("DC=example.com"); final KeyPair rootCaKeyPair = CertGenUtils.generateKeyPair(2048); + final List rootCaKeyUsages = List.of("keyCertSign", "cRLSign"); final X509Certificate rootCaCert = CertGenUtils.generateSignedCertificate( rootCaPrincipal, null, @@ -112,12 +151,15 @@ public void testIssuerCertSubjectDN() throws Exception { true, notBefore, notAfter, - null + null, + buildKeyUsage(rootCaKeyUsages), + Set.of() ); // sub CA final X500Principal subCaPrincipal = new X500Principal("DC=Sub CA,DC=example.com"); final KeyPair subCaKeyPair = CertGenUtils.generateKeyPair(2048); + final List subCaKeyUsage = List.of("digitalSignature", "keyCertSign", "cRLSign"); final X509Certificate subCaCert = CertGenUtils.generateSignedCertificate( subCaPrincipal, null, @@ -127,12 +169,15 @@ public void testIssuerCertSubjectDN() throws Exception { true, notBefore, notAfter, - null + null, + buildKeyUsage(subCaKeyUsage), + Set.of() ); // end entity final X500Principal endEntityPrincipal = new X500Principal("CN=TLS Client\\+Server,DC=Sub CA,DC=example.com"); final KeyPair endEntityKeyPair = CertGenUtils.generateKeyPair(2048); + final List endEntityKeyUsage = randomBoolean() ? null : List.of("digitalSignature", "keyEncipherment"); final X509Certificate endEntityCert = CertGenUtils.generateSignedCertificate( endEntityPrincipal, null, @@ -143,6 +188,7 @@ public void testIssuerCertSubjectDN() throws Exception { notBefore, notAfter, null, + buildKeyUsage(endEntityKeyUsage), Set.of(new ExtendedKeyUsage(KeyPurposeId.anyExtendedKeyUsage)) ); @@ -162,6 +208,101 @@ public void testIssuerCertSubjectDN() throws Exception { trustStore.setCertificateEntry("trustAnchor", rootCaCert); // anchor: any part of the chain, or issuer of last entry in chain validateEndEntityTlsChain(trustStore, certChain, true, true); + + // verify custom key usages + assertExpectedKeyUsage(rootCaCert, rootCaKeyUsages); + assertExpectedKeyUsage(subCaCert, subCaKeyUsage); + // when key usage is not specified, the key usage bits should be null + if (endEntityKeyUsage == null) { + assertThat(endEntityCert.getKeyUsage(), is(nullValue())); + assertThat(endEntityCert.getCriticalExtensionOIDs().contains(KeyUsageExtension.KEY_USAGE_OID.toString()), is(false)); + } else { + assertExpectedKeyUsage(endEntityCert, endEntityKeyUsage); + } + + } + + public void testBuildKeyUsage() { + // sanity check that lookup maps are containing the same keyUsage entries + assertThat(KEY_USAGE_BITS.keySet(), containsInAnyOrder(KEY_USAGE_MAPPINGS.keySet().toArray())); + + // passing null or empty list of keyUsage names should return null + assertThat(buildKeyUsage(null), is(nullValue())); + assertThat(buildKeyUsage(List.of()), is(nullValue())); + + // invalid names should throw IAE + var e = expectThrows(IllegalArgumentException.class, () -> buildKeyUsage(List.of(randomAlphaOfLengthBetween(3, 5)))); + assertThat(e.getMessage(), containsString("Unknown keyUsage")); + + { + final List keyUsages = randomNonEmptySubsetOf(KEY_USAGE_MAPPINGS.keySet()); + final KeyUsage keyUsage = buildKeyUsage(keyUsages); + for (String usageName : keyUsages) { + final Integer usage = KEY_USAGE_MAPPINGS.get(usageName); + assertThat(" mapping for keyUsage [" + usageName + "] is missing", usage, is(notNullValue())); + assertThat("expected keyUsage [" + usageName + "] to be set in [" + keyUsage + "]", keyUsage.hasUsages(usage), is(true)); + } + + final Set keyUsagesNotSet = KEY_USAGE_MAPPINGS.keySet() + .stream() + .filter(u -> keyUsages.contains(u) == false) + .collect(Collectors.toSet()); + + for (String usageName : keyUsagesNotSet) { + final Integer usage = KEY_USAGE_MAPPINGS.get(usageName); + assertThat(" mapping for keyUsage [" + usageName + "] is missing", usage, is(notNullValue())); + assertThat( + "expected keyUsage [" + usageName + "] not to be set in [" + keyUsage + "]", + keyUsage.hasUsages(usage), + is(false) + ); + } + + } + + { + // test that duplicates and whitespaces are ignored + KeyUsage keyUsage = buildKeyUsage( + List.of("digitalSignature ", " nonRepudiation", "\tkeyEncipherment", "keyEncipherment\n") + ); + assertThat(keyUsage.hasUsages(KEY_USAGE_MAPPINGS.get("digitalSignature")), is(true)); + assertThat(keyUsage.hasUsages(KEY_USAGE_MAPPINGS.get("nonRepudiation")), is(true)); + assertThat(keyUsage.hasUsages(KEY_USAGE_MAPPINGS.get("digitalSignature")), is(true)); + assertThat(keyUsage.hasUsages(KEY_USAGE_MAPPINGS.get("keyEncipherment")), is(true)); + } + } + + public void testIsValidKeyUsage() { + assertThat(isValidKeyUsage(randomFrom(KEY_USAGE_MAPPINGS.keySet())), is(true)); + assertThat(isValidKeyUsage(randomAlphaOfLengthBetween(3, 5)), is(false)); + + // keyUsage names are case-sensitive + assertThat(isValidKeyUsage("DigitalSignature"), is(false)); + + // white-spaces are ignored + assertThat(isValidKeyUsage("keyAgreement "), is(true)); + assertThat(isValidKeyUsage("keyCertSign\n"), is(true)); + assertThat(isValidKeyUsage("\tcRLSign "), is(true)); + } + + public static void assertExpectedKeyUsage(X509Certificate certificate, List expectedKeyUsage) { + final boolean[] keyUsage = certificate.getKeyUsage(); + assertThat("Expected " + KEY_USAGE_BITS.size() + " bits for key usage", keyUsage.length, equalTo(KEY_USAGE_BITS.size())); + final Set expectedBitsToBeSet = expectedKeyUsage.stream().map(KEY_USAGE_BITS::get).collect(Collectors.toSet()); + + for (int i = 0; i < keyUsage.length; i++) { + if (expectedBitsToBeSet.contains(i)) { + assertThat("keyUsage bit [" + i + "] expected to be set: " + expectedKeyUsage, keyUsage[i], equalTo(true)); + } else { + assertThat("keyUsage bit [" + i + "] not expected to be set: " + expectedKeyUsage, keyUsage[i], equalTo(false)); + } + } + // key usage must be marked as critical + assertThat( + "keyUsage extension should be marked as critical", + certificate.getCriticalExtensionOIDs().contains(KeyUsageExtension.KEY_USAGE_OID.toString()), + is(true) + ); } /** diff --git a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java index 1faabcfd46fdb..69ba80c729254 100644 --- a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java +++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java @@ -274,7 +274,7 @@ public void testGeneratingSignedCertificates() throws Exception { final int keysize = randomFrom(1024, 2048); final int days = randomIntBetween(1, 1024); KeyPair keyPair = CertGenUtils.generateKeyPair(keysize); - X509Certificate caCert = CertGenUtils.generateCACertificate(new X500Principal("CN=test ca"), keyPair, days); + X509Certificate caCert = CertGenUtils.generateCACertificate(new X500Principal("CN=test ca"), keyPair, days, null); final boolean generatedCa = randomBoolean(); final char[] keyPassword = randomBoolean() ? SecuritySettingsSourceField.TEST_PASSWORD.toCharArray() : null; diff --git a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java index 1a11234c98e6e..32b2aabb29611 100644 --- a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java +++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java @@ -415,7 +415,13 @@ public void testGeneratingSignedPemCertificates() throws Exception { int days = randomIntBetween(1, 1024); KeyPair keyPair = CertGenUtils.generateKeyPair(keySize); - X509Certificate caCert = CertGenUtils.generateCACertificate(new X500Principal("CN=test ca"), keyPair, days); + List caKeyUsage = randomBoolean() ? null : CertificateTool.DEFAULT_CA_KEY_USAGE; + X509Certificate caCert = CertGenUtils.generateCACertificate( + new X500Principal("CN=test ca"), + keyPair, + days, + CertGenUtils.buildKeyUsage(caKeyUsage) + ); final boolean selfSigned = randomBoolean(); final String keyPassword = randomBoolean() ? SecuritySettingsSourceField.TEST_PASSWORD : null; @@ -1191,6 +1197,7 @@ private String generateCA(Path caFile, MockTerminal terminal, Environment env, b final int caKeySize = randomIntBetween(4, 8) * 512; final int days = randomIntBetween(7, 1500); final String caPassword = randomFrom("", randomAlphaOfLengthBetween(4, 80)); + final String caKeyUsage = randomFrom("", Strings.collectionToCommaDelimitedString(CertificateTool.DEFAULT_CA_KEY_USAGE)); final CertificateAuthorityCommand caCommand = new PathAwareCertificateAuthorityCommand(caFile); String[] args = { @@ -1203,7 +1210,9 @@ private String generateCA(Path caFile, MockTerminal terminal, Environment env, b "-keysize", String.valueOf(caKeySize), "-days", - String.valueOf(days) }; + String.valueOf(days), + "-keyusage", + caKeyUsage }; if (pem) { args = ArrayUtils.append(args, "--pem"); } diff --git a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommandTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommandTests.java index 1033d4e51ebba..57ad76af3317a 100644 --- a/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommandTests.java +++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/HttpCertificateCommandTests.java @@ -23,6 +23,7 @@ import org.bouncycastle.asn1.x509.GeneralName; import org.bouncycastle.asn1.x509.GeneralNames; import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.asn1.x509.KeyUsage; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest; import org.bouncycastle.util.io.pem.PemObject; @@ -30,6 +31,7 @@ import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.ProcessInfo; import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.ssl.PemUtils; import org.elasticsearch.core.CheckedFunction; @@ -89,10 +91,12 @@ import static org.elasticsearch.test.FileMatchers.isDirectory; import static org.elasticsearch.test.FileMatchers.isRegularFile; import static org.elasticsearch.test.FileMatchers.pathExists; +import static org.elasticsearch.xpack.security.cli.CertGenUtilsTests.assertExpectedKeyUsage; import static org.elasticsearch.xpack.security.cli.HttpCertificateCommand.guessFileType; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.instanceOf; @@ -369,21 +373,25 @@ public void testGenerateMultipleCertificateWithNewCA() throws Exception { final String caDN; final int caYears; final int caKeySize; + final List caKeyUsage; // randomise whether to change CA defaults. if (randomBoolean()) { terminal.addTextInput("y"); // Change defaults caDN = "CN=" + randomAlphaOfLengthBetween(3, 8); caYears = randomIntBetween(1, 3); caKeySize = randomFrom(2048, 3072, 4096); + caKeyUsage = randomNonEmptySubsetOf(CertGenUtils.KEY_USAGE_MAPPINGS.keySet()); terminal.addTextInput(caDN); terminal.addTextInput(caYears + "y"); terminal.addTextInput(Integer.toString(caKeySize)); + terminal.addTextInput(Strings.collectionToCommaDelimitedString(caKeyUsage)); terminal.addTextInput("n"); // Don't change values } else { terminal.addTextInput(randomBoolean() ? "n" : ""); // Don't change defaults caDN = HttpCertificateCommand.DEFAULT_CA_NAME.toString(); caYears = HttpCertificateCommand.DEFAULT_CA_VALIDITY.getYears(); caKeySize = HttpCertificateCommand.DEFAULT_CA_KEY_SIZE; + caKeyUsage = HttpCertificateCommand.DEFAULT_CA_KEY_USAGE; } final String caPassword = randomPassword(randomBoolean()); @@ -463,6 +471,7 @@ public void testGenerateMultipleCertificateWithNewCA() throws Exception { verifyCertificate(caCertKey.v1(), caDN.replaceFirst("CN=", ""), caYears, List.of(), List.of()); assertThat(getRSAKeySize(caCertKey.v1().getPublicKey()), is(caKeySize)); assertThat(getRSAKeySize(caCertKey.v2()), is(caKeySize)); + assertExpectedKeyUsage(caCertKey.v1(), caKeyUsage); assertThat(zipRoot.resolve("elasticsearch"), isDirectory()); @@ -486,6 +495,7 @@ public void testGenerateMultipleCertificateWithNewCA() throws Exception { verifyChain(certAndKey.v1(), caCertKey.v1()); assertThat(getRSAKeySize(certAndKey.v1().getPublicKey()), is(HttpCertificateCommand.DEFAULT_CERT_KEY_SIZE)); assertThat(getRSAKeySize(certAndKey.v2()), is(HttpCertificateCommand.DEFAULT_CERT_KEY_SIZE)); + assertExpectedKeyUsage(certAndKey.v1(), HttpCertificateCommand.DEFAULT_CERT_KEY_USAGE); // Verify the README assertThat(readme, containsString(p12Path.getFileName().toString())); @@ -692,7 +702,10 @@ private void verifyCertificationRequest( // We register 1 extension with the subject alternative names and extended key usage final Extensions extensions = Extensions.getInstance(extensionAttributes[0].getAttributeValues()[0]); assertThat(extensions, notNullValue()); - assertThat(extensions.getExtensionOIDs(), arrayWithSize(2)); + assertThat( + extensions.getExtensionOIDs(), + arrayContainingInAnyOrder(Extension.subjectAlternativeName, Extension.keyUsage, Extension.extendedKeyUsage) + ); final GeneralNames names = GeneralNames.fromExtensions(extensions, Extension.subjectAlternativeName); assertThat(names.getNames(), arrayWithSize(hostNames.size() + ipAddresses.size())); @@ -709,6 +722,9 @@ private void verifyCertificationRequest( ExtendedKeyUsage extendedKeyUsage = ExtendedKeyUsage.fromExtensions(extensions); assertThat(extendedKeyUsage.getUsages(), arrayContainingInAnyOrder(KeyPurposeId.id_kp_serverAuth)); + + KeyUsage keyUsage = KeyUsage.fromExtensions(extensions); + assertThat(keyUsage, is(equalTo(new KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyEncipherment)))); } private void verifyCertificate( diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle index f53ff7027f126..4855a9286a7eb 100644 --- a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle +++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle @@ -11,7 +11,7 @@ apply plugin: 'com.gradleup.shadow' // See the build.gradle file in the parent directory for an explanation of this unusual build dependencies { - implementation "com.nimbusds:nimbus-jose-jwt:9.37.3" + implementation "com.nimbusds:nimbus-jose-jwt:10.0.2" } tasks.named('shadowJar').configure { diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle index 4418bd32e64cf..d83788891845d 100644 --- a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle +++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle @@ -11,7 +11,7 @@ apply plugin: 'com.gradleup.shadow' // See the build.gradle file in the parent directory for an explanation of this unusual build dependencies { - implementation "com.nimbusds:nimbus-jose-jwt:9.37.3" + implementation "com.nimbusds:nimbus-jose-jwt:10.0.2" implementation project(path: xpackModule('security:lib:nimbus-jose-jwt-modified-part2'), configuration: 'shadow') } diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java index 1ea11f5c280ef..34b61e612c747 100644 --- a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java +++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java @@ -13,6 +13,7 @@ import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.text.ParseException; +import java.util.Date; import java.util.List; import java.util.Map; @@ -192,6 +193,16 @@ public static Base64URL getBase64URL(final Map o, final String k } } + public static Date getEpochSecondAsDate(final Map o, final String key) throws ParseException { + try { + return AccessController.doPrivileged( + (PrivilegedExceptionAction) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getEpochSecondAsDate(o, key) + ); + } catch (PrivilegedActionException e) { + throw (ParseException) e.getException(); + } + } + public static String toJSONString(final Map o) { return AccessController.doPrivileged( (PrivilegedAction) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.toJSONString(o) diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 274587318d555..b66bacf71dd60 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.security.operator; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.common.util.FeatureFlag; import java.util.Objects; import java.util.Set; @@ -327,6 +326,8 @@ public class Constants { "cluster:admin/xpack/watcher/settings/update", "cluster:admin/xpack/watcher/watch/put", "cluster:internal/remote_cluster/nodes", + "cluster:internal/xpack/inference", + "cluster:internal/xpack/inference/unified", "cluster:internal/xpack/ml/coordinatedinference", "cluster:internal/xpack/ml/datafeed/isolate", "cluster:internal/xpack/ml/datafeed/running_state", @@ -386,9 +387,8 @@ public class Constants { "cluster:monitor/xpack/enrich/stats", "cluster:monitor/xpack/eql/stats/dist", "cluster:monitor/xpack/esql/stats/dist", - "cluster:monitor/xpack/inference", + "cluster:monitor/xpack/inference/post", "cluster:monitor/xpack/inference/get", - "cluster:monitor/xpack/inference/unified", "cluster:monitor/xpack/inference/diagnostics/get", "cluster:monitor/xpack/inference/services/get", "cluster:monitor/xpack/info", @@ -639,11 +639,12 @@ public class Constants { "internal:gateway/local/started_shards", "internal:admin/indices/prevalidate_shard_path", "internal:index/metadata/migration_version/update", - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/migration/reindex_status" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/index/reindex" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex_cancel" : null, - new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/index/create_from_source" : null, + "indices:admin/migration/reindex_status", + "indices:admin/data_stream/index/reindex", + "indices:admin/data_stream/reindex", + "indices:admin/data_stream/reindex_cancel", + "indices:admin/index/create_from_source", + "indices:admin/index/copy_lifecycle_index_metadata", "internal:admin/repository/verify", "internal:admin/repository/verify/coordinate" ).filter(Objects::nonNull).collect(Collectors.toUnmodifiableSet()); diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/GetRolesIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/GetRolesIT.java new file mode 100644 index 0000000000000..c8499aa3a9eba --- /dev/null +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/GetRolesIT.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.local.model.User; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.xcontent.ObjectPath; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; + +public class GetRolesIT extends SecurityInBasicRestTestCase { + + private static final String ADMIN_USER = "admin_user"; + private static final SecureString ADMIN_PASSWORD = new SecureString("admin-password".toCharArray()); + protected static final String READ_SECURITY_USER = "read_security_user"; + private static final SecureString READ_SECURITY_PASSWORD = new SecureString("read-security-password".toCharArray()); + + @Before + public void initialize() { + new ReservedRolesStore(); + } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .nodes(2) + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "basic") + .rolesFile(Resource.fromClasspath("roles.yml")) + .user(ADMIN_USER, ADMIN_PASSWORD.toString(), User.ROOT_USER_ROLE, true) + .user(READ_SECURITY_USER, READ_SECURITY_PASSWORD.toString(), "read_security_user_role", false) + .build(); + + @Override + protected Settings restAdminSettings() { + String token = basicAuthHeaderValue(ADMIN_USER, ADMIN_PASSWORD); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(READ_SECURITY_USER, READ_SECURITY_PASSWORD); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testGetAllRolesNoNative() throws Exception { + // Test get roles API with operator admin_user + getAllRolesAndAssert(adminClient(), ReservedRolesStore.names()); + // Test get roles API with read_security_user + getAllRolesAndAssert(client(), ReservedRolesStore.names()); + } + + public void testGetAllRolesWithNative() throws Exception { + createRole("custom_role", "Test custom native role.", Map.of("owner", "test")); + + Set expectedRoles = new HashSet<>(ReservedRolesStore.names()); + expectedRoles.add("custom_role"); + + // Test get roles API with operator admin_user + getAllRolesAndAssert(adminClient(), expectedRoles); + // Test get roles API with read_security_user + getAllRolesAndAssert(client(), expectedRoles); + } + + public void testGetReservedOnly() throws Exception { + createRole("custom_role", "Test custom native role.", Map.of("owner", "test")); + + Set rolesToGet = new HashSet<>(); + rolesToGet.add("custom_role"); + rolesToGet.addAll(randomSet(1, 5, () -> randomFrom(ReservedRolesStore.names()))); + + getRolesAndAssert(adminClient(), rolesToGet); + getRolesAndAssert(client(), rolesToGet); + } + + public void testGetNativeOnly() throws Exception { + createRole("custom_role1", "Test custom native role.", Map.of("owner", "test1")); + createRole("custom_role2", "Test custom native role.", Map.of("owner", "test2")); + + Set rolesToGet = Set.of("custom_role1", "custom_role2"); + + getRolesAndAssert(adminClient(), rolesToGet); + getRolesAndAssert(client(), rolesToGet); + } + + public void testGetMixedRoles() throws Exception { + createRole("custom_role", "Test custom native role.", Map.of("owner", "test")); + + Set rolesToGet = new HashSet<>(); + rolesToGet.add("custom_role"); + rolesToGet.addAll(randomSet(1, 5, () -> randomFrom(ReservedRolesStore.names()))); + + getRolesAndAssert(adminClient(), rolesToGet); + getRolesAndAssert(client(), rolesToGet); + } + + public void testNonExistentRole() { + var e = expectThrows( + ResponseException.class, + () -> client().performRequest(new Request("GET", "/_security/role/non_existent_role")) + ); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + } + + private void createRole(String roleName, String description, Map metadata) throws IOException { + Request request = new Request("POST", "/_security/role/" + roleName); + Map requestMap = new HashMap<>(); + if (description != null) { + requestMap.put(RoleDescriptor.Fields.DESCRIPTION.getPreferredName(), description); + } + if (metadata != null) { + requestMap.put(RoleDescriptor.Fields.METADATA.getPreferredName(), metadata); + } + BytesReference source = BytesReference.bytes(jsonBuilder().map(requestMap)); + request.setJsonEntity(source.utf8ToString()); + Response response = adminClient().performRequest(request); + assertOK(response); + Map responseMap = responseAsMap(response); + assertTrue(ObjectPath.eval("role.created", responseMap)); + } + + private void getAllRolesAndAssert(RestClient client, Set expectedRoles) throws IOException { + final Response response = client.performRequest(new Request("GET", "/_security/role")); + assertOK(response); + final Map responseMap = responseAsMap(response); + assertThat(responseMap.keySet(), equalTo(expectedRoles)); + } + + private void getRolesAndAssert(RestClient client, Set rolesToGet) throws IOException { + final Response response = client.performRequest(new Request("GET", "/_security/role/" + String.join(",", rolesToGet))); + assertOK(response); + final Map responseMap = responseAsMap(response); + assertThat(responseMap.keySet(), equalTo(rolesToGet)); + } +} diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java index 7adff21d8df4f..e8363cad4a68b 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java @@ -121,14 +121,16 @@ public void testQueryDeleteOrUpdateReservedRoles() throws Exception { waitForMigrationCompletion(adminClient(), SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION); final String[] allReservedRoles = ReservedRolesStore.names().toArray(new String[0]); - assertQuery(client(), """ - { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 } - """, allReservedRoles.length, roles -> { - assertThat(roles, iterableWithSize(allReservedRoles.length)); - for (var role : roles) { - assertThat((String) role.get("name"), is(oneOf(allReservedRoles))); - } - }); + assertBusy(() -> { + assertQuery(client(), """ + { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 } + """, allReservedRoles.length, roles -> { + assertThat(roles, iterableWithSize(allReservedRoles.length)); + for (var role : roles) { + assertThat((String) role.get("name"), is(oneOf(allReservedRoles))); + } + }); + }, 30, TimeUnit.SECONDS); final String roleName = randomFrom(allReservedRoles); assertQuery(client(), String.format(""" diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/ssl/SslEntitlementRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/ssl/SslEntitlementRestIT.java new file mode 100644 index 0000000000000..f661bb04dc3da --- /dev/null +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/ssl/SslEntitlementRestIT.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.ssl; + +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.LogType; +import org.elasticsearch.test.cluster.MutableSettingsProvider; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.is; + +public class SslEntitlementRestIT extends ESRestTestCase { + + private static final MutableSettingsProvider settingsProvider = new MutableSettingsProvider(); + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .apply(SecurityOnTrialLicenseRestTestCase.commonTrialSecurityClusterConfig) + .settings(settingsProvider) + .systemProperty("es.entitlements.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testSslEntitlementInaccessiblePath() throws IOException { + settingsProvider.put("xpack.security.transport.ssl.key", "/bad/path/transport.key"); + settingsProvider.put("xpack.security.transport.ssl.certificate", "/bad/path/transport.crt"); + expectThrows(Exception.class, () -> cluster.restart(false)); + AtomicBoolean found = new AtomicBoolean(false); + for (int i = 0; i < cluster.getNumNodes(); i++) { + try (InputStream log = cluster.getNodeLog(i, LogType.SERVER)) { + Streams.readAllLines(log, line -> { + if (line.contains("failed to load SSL configuration") && line.contains("because access to read the file is blocked")) { + found.set(true); + } + }); + } + } + assertThat("cluster logs did not include events of blocked file access", found.get(), is(true)); + } + + @Override + protected Settings restAdminSettings() { + String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + // as the cluster is dead its state can not be wiped successfully so we have to bypass wiping the cluster + return true; + } +} diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 458dee693c80a..9c3cabf8a74ce 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -447,6 +447,7 @@ public Collection getSystemDataStreamDescriptors() { .build(), Map.of(), Collections.singletonList("test"), + "test", new ExecutorNames( ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java index 2c98d2e686e46..2afacbd56d22b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java @@ -276,6 +276,7 @@ public Collection getSystemDataStreamDescriptors() { .build(), Map.of(), Collections.singletonList("test"), + "test", new ExecutorNames(ThreadPool.Names.SYSTEM_CRITICAL_READ, ThreadPool.Names.SYSTEM_READ, ThreadPool.Names.SYSTEM_WRITE) ) ); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/GeRoleDescriptorsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/GeRoleDescriptorsTests.java new file mode 100644 index 0000000000000..3e7312f67b08f --- /dev/null +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/GeRoleDescriptorsTests.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.integration; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.test.NativeRealmIntegTestCase; +import org.elasticsearch.test.TestSecurityClient; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; +import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; +import org.elasticsearch.xpack.security.support.SecuritySystemIndices; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Test for the {@link NativeRolesStore#getRoleDescriptors} method. + */ +public class GeRoleDescriptorsTests extends NativeRealmIntegTestCase { + + private static Set customRoles; + + @BeforeClass + public static void init() throws Exception { + new ReservedRolesStore(); + + final int numOfRoles = randomIntBetween(5, 10); + customRoles = new HashSet<>(numOfRoles); + for (int i = 0; i < numOfRoles; i++) { + customRoles.add("custom_role_" + randomAlphaOfLength(10) + "_" + i); + } + } + + @Before + public void setup() throws IOException { + final TestSecurityClient securityClient = new TestSecurityClient(getRestClient(), SECURITY_REQUEST_OPTIONS); + for (String role : customRoles) { + final RoleDescriptor descriptor = new RoleDescriptor( + role, + new String[0], + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices("*") + .privileges("ALL") + .allowRestrictedIndices(randomBoolean()) + .build() }, + new String[0] + ); + securityClient.putRole(descriptor); + logger.info("--> created role [{}]", role); + } + + ensureGreen(SecuritySystemIndices.SECURITY_MAIN_ALIAS); + } + + public void testGetCustomRoles() { + for (NativeRolesStore rolesStore : internalCluster().getInstances(NativeRolesStore.class)) { + PlainActionFuture future = new PlainActionFuture<>(); + rolesStore.getRoleDescriptors(customRoles, future); + RoleRetrievalResult result = future.actionGet(); + assertThat(result, notNullValue()); + assertTrue(result.isSuccess()); + assertThat(result.getDescriptors().stream().map(RoleDescriptor::getName).toList(), containsInAnyOrder(customRoles.toArray())); + } + } + + public void testGetReservedRoles() { + for (NativeRolesStore rolesStore : internalCluster().getInstances(NativeRolesStore.class)) { + PlainActionFuture future = new PlainActionFuture<>(); + Set reservedRoles = randomUnique(() -> randomFrom(ReservedRolesStore.names()), randomIntBetween(1, 5)); + AssertionError error = expectThrows(AssertionError.class, () -> rolesStore.getRoleDescriptors(reservedRoles, future)); + assertThat(error.getMessage(), containsString("native roles store should not be called with reserved role names")); + } + } + + public void testGetAllRoles() { + for (NativeRolesStore rolesStore : internalCluster().getInstances(NativeRolesStore.class)) { + PlainActionFuture future = new PlainActionFuture<>(); + rolesStore.getRoleDescriptors(randomBoolean() ? null : Set.of(), future); + RoleRetrievalResult result = future.actionGet(); + assertThat(result, notNullValue()); + assertTrue(result.isSuccess()); + assertThat(result.getDescriptors().stream().map(RoleDescriptor::getName).toList(), containsInAnyOrder(customRoles.toArray())); + } + } +} diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java index 07bdd83c9a144..53d2ebda1fee1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java @@ -7,9 +7,16 @@ package org.elasticsearch.test; import org.apache.http.HttpHost; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; @@ -27,6 +34,7 @@ import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; import org.elasticsearch.xpack.security.LocalStateSecurity; import org.elasticsearch.xpack.security.support.SecurityMigrations; @@ -45,9 +53,12 @@ import java.util.stream.Collectors; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.getMigrationVersionFromIndexMetadata; +import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; /** * A test that starts a single node with security enabled. This test case allows for customization @@ -82,6 +93,13 @@ public static void destroyDefaultSettings() { tearDownRestClient(); } + @Override + public void setUp() throws Exception { + super.setUp(); + deleteSecurityIndexIfExists(); + createSecurityIndexWithWaitForActiveShards(); + } + @Override public void tearDown() throws Exception { awaitSecurityMigration(); @@ -100,7 +118,7 @@ private boolean isMigrationComplete(ClusterState state) { return getMigrationVersionFromIndexMetadata(indexMetadata) == SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey(); } - private void awaitSecurityMigration() { + protected void awaitSecurityMigration() { final var latch = new CountDownLatch(1); ClusterService clusterService = getInstanceFromNode(ClusterService.class); clusterService.addListener((event) -> { @@ -362,4 +380,40 @@ private static RestClient createRestClient( } return builder.build(); } + + protected void deleteSecurityIndexIfExists() { + // delete the security index, if it exist + GetIndexRequest getIndexRequest = new GetIndexRequest(); + getIndexRequest.indices(SECURITY_MAIN_ALIAS); + getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet(); + if (getIndexResponse.getIndices().length > 0) { + assertThat(getIndexResponse.getIndices().length, is(1)); + assertThat(getIndexResponse.getIndices()[0], is(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7)); + + // Security migration needs to finish before deleting the index + awaitSecurityMigration(); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices()); + assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet()); + } + } + + protected void createSecurityIndexWithWaitForActiveShards() { + final Client client = client().filterWithHeader( + Collections.singletonMap( + "Authorization", + UsernamePasswordToken.basicAuthHeaderValue( + SecuritySettingsSource.ES_TEST_ROOT_USER, + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING + ) + ) + ); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS).waitForActiveShards(ActiveShardCount.ALL) + .masterNodeTimeout(TEST_REQUEST_TIMEOUT); + try { + client.admin().indices().create(createIndexRequest).actionGet(); + } catch (ResourceAlreadyExistsException e) { + logger.info("Security index already exists, ignoring.", e); + } + } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java index c714aa352fd41..5a76b81a9f3fc 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.SecurityIntegTestCase; +import java.io.IOException; + import static org.elasticsearch.test.NodeRoles.dataOnlyNode; import static org.elasticsearch.test.NodeRoles.masterNode; import static org.hamcrest.Matchers.containsString; @@ -18,15 +20,19 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) public class UnregisteredSettingsIntegTests extends SecurityIntegTestCase { - public void testIncludeReservedRolesSettingNotRegistered() { + public void testIncludeReservedRolesSettingNotRegistered() throws IOException { internalCluster().setBootstrapMasterNodeIndex(0); final Settings.Builder builder = Settings.builder() .put(randomBoolean() ? masterNode() : dataOnlyNode()) .putList("xpack.security.reserved_roles.include", "superuser"); - final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder)); - assertThat(e.getMessage(), containsString("unknown setting [xpack.security.reserved_roles.include]")); + try { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder)); + assertThat(e.getMessage(), containsString("unknown setting [xpack.security.reserved_roles.include]")); + } finally { + internalCluster().close(); + } } public void testSamlExcludeRolesSettingNotRegistered() throws Exception { @@ -36,7 +42,11 @@ public void testSamlExcludeRolesSettingNotRegistered() throws Exception { .put(randomBoolean() ? masterNode() : dataOnlyNode()) .putList("xpack.security.authc.realms.saml.saml1.exclude_roles", "superuser"); - final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder)); - assertThat(e.getMessage(), containsString("unknown setting [xpack.security.authc.realms.saml.saml1.exclude_roles]")); + try { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder)); + assertThat(e.getMessage(), containsString("unknown setting [xpack.security.authc.realms.saml.saml1.exclude_roles]")); + } finally { + internalCluster().close(); + } } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java index 825ae68225627..8b8e6fa6b8ea8 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java @@ -8,17 +8,10 @@ package org.elasticsearch.xpack.security.authc.esnative; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; @@ -29,14 +22,9 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; import org.junit.BeforeClass; -import java.util.concurrent.CountDownLatch; - import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY; -import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.is; public class ReservedRealmElasticAutoconfigIntegTests extends SecuritySingleNodeTestCase { @@ -70,46 +58,10 @@ protected SecureString getBootstrapPassword() { return null; // no bootstrap password for this test } - private boolean isMigrationComplete(ClusterState state) { - IndexMetadata indexMetadata = state.metadata().getIndices().get(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7); - return indexMetadata != null && indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) != null; - } - - private void awaitSecurityMigrationRanOnce() { - final var latch = new CountDownLatch(1); - ClusterService clusterService = getInstanceFromNode(ClusterService.class); - clusterService.addListener((event) -> { - if (isMigrationComplete(event.state())) { - latch.countDown(); - } - }); - if (isMigrationComplete(clusterService.state())) { - latch.countDown(); - } - safeAwait(latch); - } - - private void deleteSecurityIndex() { - // delete the security index, if it exist - GetIndexRequest getIndexRequest = new GetIndexRequest(); - getIndexRequest.indices(SECURITY_MAIN_ALIAS); - getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); - GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet(); - if (getIndexResponse.getIndices().length > 0) { - assertThat(getIndexResponse.getIndices().length, is(1)); - assertThat(getIndexResponse.getIndices()[0], is(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7)); - - // Security migration needs to finish before deleting the index - awaitSecurityMigrationRanOnce(); - DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices()); - assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet()); - } - } - public void testAutoconfigFailedPasswordPromotion() throws Exception { try { // .security index is created automatically on node startup so delete the security index first - deleteSecurityIndex(); + deleteSecurityIndexIfExists(); // prevents the .security index from being created automatically (after elastic user authentication) ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest( TEST_REQUEST_TIMEOUT, @@ -176,7 +128,7 @@ public void testAutoconfigSucceedsAfterPromotionFailure() throws Exception { putUserRequest.roles(Strings.EMPTY_ARRAY); client().execute(PutUserAction.INSTANCE, putUserRequest).get(); // Security migration needs to finish before making the cluster read only - awaitSecurityMigrationRanOnce(); + awaitSecurityMigration(); // but then make the cluster read-only ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest( diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java index 8324763c57bd2..407c6fba05220 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java @@ -802,7 +802,7 @@ public void testRemoveIndex() { assertAcked(client.admin().indices().prepareAliases().removeIndex("*").get()); GetAliasesResponse getAliasesResponse = client.admin().indices().prepareGetAliases().setAliases("*").get(); assertThat(getAliasesResponse.getAliases().size(), equalTo(0)); - assertAliases(indicesAdmin().prepareGetAliases().setAliases("*"), "bogus_index_1", "bogus_alias_1", "bogus_alias_2"); + assertAliases(indicesAdmin().prepareGetAliases().setAliases("*", "-.security*"), "bogus_index_1", "bogus_alias_1", "bogus_alias_2"); } public void testAliasesForHiddenIndices() { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java index f2a91cb1b8e4e..21c4d8d99a14d 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java @@ -99,13 +99,28 @@ public class ProfileIntegTests extends AbstractProfileIntegTestCase { + protected static final String ANONYMOUS_ROLE = "anonymous_role"; + + @Override + protected String configRoles() { + return super.configRoles() + + "\n" + + ANONYMOUS_ROLE + + ":\n" + + " cluster:\n" + + " - 'manage_own_api_key'\n" + + " - 'manage_token'\n" + + " - 'manage_service_account'\n" + + " - 'monitor'\n"; + } + @Override protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { final Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); // This setting tests that the setting is registered builder.put("xpack.security.authc.domains.my_domain.realms", "file"); // enable anonymous - builder.putList(AnonymousUser.ROLES_SETTING.getKey(), RAC_ROLE); + builder.putList(AnonymousUser.ROLES_SETTING.getKey(), ANONYMOUS_ROLE); return builder.build(); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadDuringStartupIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadDuringStartupIntegTests.java index 44f7a6d47e361..b1fda5f6c4e6e 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadDuringStartupIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadDuringStartupIntegTests.java @@ -55,7 +55,7 @@ public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Environment tmpEnv = TestEnvironment.newEnvironment(settings); // For each node, copy the original testnode.jks into each node's config directory. - Path nodeKeystorePath = tmpEnv.configFile().resolve("testnode.jks"); + Path nodeKeystorePath = tmpEnv.configDir().resolve("testnode.jks"); try { Path goodKeystorePath = getDataPath(goodKeyStoreFilePath); Files.copy(goodKeystorePath, nodeKeystorePath, StandardCopyOption.REPLACE_EXISTING); @@ -93,7 +93,7 @@ public void testReloadDuringStartup() throws Exception { final Environment env = internalCluster().getInstance(Environment.class, nodeName); final CountDownLatch beforeKeystoreFix = new CountDownLatch(2); // SYNC: Cert update & ES restart final CountDownLatch afterKeystoreFix = new CountDownLatch(1); // SYNC: Verify cluster after cert update - final Path nodeKeystorePath = env.configFile().resolve("testnode.jks"); // all nodes have good keystore + final Path nodeKeystorePath = env.configDir().resolve("testnode.jks"); // all nodes have good keystore final Path badKeystorePath = getDataPath(badKeyStoreFilePath); // stop a node, and apply this bad keystore final Path goodKeystorePath = getDataPath(goodKeyStoreFilePath); // start the node, and apply this good keystore assertTrue(Files.exists(nodeKeystorePath)); diff --git a/x-pack/plugin/security/src/main/java/module-info.java b/x-pack/plugin/security/src/main/java/module-info.java index 947211559b0c2..a2798faefaa38 100644 --- a/x-pack/plugin/security/src/main/java/module-info.java +++ b/x-pack/plugin/security/src/main/java/module-info.java @@ -49,6 +49,7 @@ requires oauth2.oidc.sdk; requires org.slf4j; requires unboundid.ldapsdk; + requires org.elasticsearch.logging; exports org.elasticsearch.xpack.security.action to org.elasticsearch.server; exports org.elasticsearch.xpack.security.action.apikey to org.elasticsearch.server; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java index 583bb93c2a52b..e6bdfd9cde14b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java @@ -10,6 +10,7 @@ import org.elasticsearch.watcher.FileWatcher; import java.io.IOException; +import java.io.InputStream; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -34,6 +35,15 @@ public PrivilegedFileWatcher(Path path) { super(path); } + public PrivilegedFileWatcher(Path path, boolean checkFileContents) { + super(path, checkFileContents); + } + + @Override + protected InputStream newInputStream(Path path) throws IOException { + return Files.newInputStream(path); + } + @Override protected boolean fileExists(Path path) { return doPrivileged((PrivilegedAction) () -> Files.exists(path)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 9279a5e870c80..6ad7e1ca2bc27 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -725,9 +725,9 @@ protected List getReloadableSecurityComponents() { * ES has already checked the file is actually in the config directory */ public static Path resolveSecuredConfigFile(Environment env, String file) { - Path config = env.configFile().resolve(file); + Path config = env.configDir().resolve(file); if (doPrivileged((PrivilegedAction) () -> Files.exists(config)) == false) { - Path legacyConfig = env.configFile().resolve("x-pack").resolve(file); + Path legacyConfig = env.configDir().resolve("x-pack").resolve(file); if (doPrivileged((PrivilegedAction) () -> Files.exists(legacyConfig))) { DeprecationLogger.getLogger(XPackPlugin.class) .warn( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java index cdeac51e1f492..38545281928b1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import java.util.Arrays; @@ -29,11 +30,18 @@ public class TransportGetRolesAction extends TransportAction { private final NativeRolesStore nativeRolesStore; + private final ReservedRoleNameChecker reservedRoleNameChecker; @Inject - public TransportGetRolesAction(ActionFilters actionFilters, NativeRolesStore nativeRolesStore, TransportService transportService) { + public TransportGetRolesAction( + ActionFilters actionFilters, + NativeRolesStore nativeRolesStore, + ReservedRoleNameChecker reservedRoleNameChecker, + TransportService transportService + ) { super(GetRolesAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.nativeRolesStore = nativeRolesStore; + this.reservedRoleNameChecker = reservedRoleNameChecker; } @Override @@ -43,9 +51,14 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL if (request.nativeOnly()) { final Set rolesToSearchFor = specificRolesRequested - ? Arrays.stream(requestedRoles).collect(Collectors.toSet()) + ? Arrays.stream(requestedRoles).filter(r -> false == reservedRoleNameChecker.isReserved(r)).collect(Collectors.toSet()) : Collections.emptySet(); - getNativeRoles(rolesToSearchFor, listener); + if (specificRolesRequested && rolesToSearchFor.isEmpty()) { + // specific roles were requested, but they were all reserved, no need to hit the native store + listener.onResponse(new GetRolesResponse()); + } else { + getNativeRoles(rolesToSearchFor, listener); + } return; } @@ -53,13 +66,10 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL final Set reservedRoles = new LinkedHashSet<>(); if (specificRolesRequested) { for (String role : requestedRoles) { - if (ReservedRolesStore.isReserved(role)) { + if (reservedRoleNameChecker.isReserved(role)) { RoleDescriptor rd = ReservedRolesStore.roleDescriptor(role); if (rd != null) { reservedRoles.add(rd); - } else { - listener.onFailure(new IllegalStateException("unable to obtain reserved role [" + role + "]")); - return; } } else { rolesToSearchFor.add(role); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordTool.java index 0718742d362cb..f04c670eb1ea7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordTool.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordTool.java @@ -43,7 +43,7 @@ class ResetPasswordTool extends BaseRunAsSuperuserCommand { private final OptionSpec usernameOption; ResetPasswordTool() { - this(CommandLineHttpClient::new, environment -> KeyStoreWrapper.load(environment.configFile())); + this(CommandLineHttpClient::new, environment -> KeyStoreWrapper.load(environment.configDir())); } protected ResetPasswordTool( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java index 91c75c076881e..3c7fa029d4514 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java @@ -95,11 +95,11 @@ class SetupPasswordTool extends MultiCommand { SetupPasswordTool() { this(environment -> new CommandLineHttpClient(environment), environment -> { - KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(environment.configFile()); + KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(environment.configDir()); if (keyStoreWrapper == null) { throw new UserException( ExitCodes.CONFIG, - "Elasticsearch keystore file is missing [" + KeyStoreWrapper.keystorePath(environment.configFile()) + "]" + "Elasticsearch keystore file is missing [" + KeyStoreWrapper.keystorePath(environment.configDir()) + "]" ); } return keyStoreWrapper; @@ -142,7 +142,7 @@ class AutoSetup extends SetupCommand { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { - terminal.println(Verbosity.VERBOSE, "Running with configuration path: " + env.configFile()); + terminal.println(Verbosity.VERBOSE, "Running with configuration path: " + env.configDir()); setupOptions(terminal, options, env); checkElasticKeystorePasswordValid(terminal, env); checkClusterHealth(terminal); @@ -198,7 +198,7 @@ class InteractiveSetup extends SetupCommand { @Override public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception { - terminal.println(Verbosity.VERBOSE, "Running with configuration path: " + env.configFile()); + terminal.println(Verbosity.VERBOSE, "Running with configuration path: " + env.configDir()); setupOptions(terminal, options, env); checkElasticKeystorePasswordValid(terminal, env); checkClusterHealth(terminal); @@ -298,7 +298,7 @@ void setupOptions(Terminal terminal, OptionSet options, Environment env) throws Settings settings = settingsBuilder.build(); elasticUserPassword = ReservedRealm.BOOTSTRAP_ELASTIC_PASSWORD.get(settings); - final Environment newEnv = new Environment(settings, env.configFile()); + final Environment newEnv = new Environment(settings, env.configDir()); Environment.assertEquivalent(newEnv, env); client = clientFunction.apply(newEnv); @@ -354,7 +354,7 @@ void checkElasticKeystorePasswordValid(Terminal terminal, Environment env) throw terminal.errorPrintln("Possible causes include:"); terminal.errorPrintln(" * The password for the '" + elasticUser + "' user has already been changed on this cluster"); terminal.errorPrintln(" * Your elasticsearch node is running against a different keystore"); - terminal.errorPrintln(" This tool used the keystore at " + KeyStoreWrapper.keystorePath(env.configFile())); + terminal.errorPrintln(" This tool used the keystore at " + KeyStoreWrapper.keystorePath(env.configDir())); terminal.errorPrintln(""); terminal.errorPrintln( "You can use the `elasticsearch-reset-password` CLI tool to reset the password of the '" + elasticUser + "' user" diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java index ffc14ca96a768..06606570699d9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.support.NoOpLogger; import org.elasticsearch.xpack.core.security.support.Validation; +import org.elasticsearch.xpack.security.PrivilegedFileWatcher; import org.elasticsearch.xpack.security.support.SecurityFiles; import java.io.IOException; @@ -57,7 +58,7 @@ public class FileUserRolesStore { file = resolveFile(config.env()); userRoles = parseFileLenient(file, logger); listeners = new CopyOnWriteArrayList<>(Collections.singletonList(listener)); - FileWatcher watcher = new FileWatcher(file.getParent()); + FileWatcher watcher = new PrivilegedFileWatcher(file.getParent()); watcher.addListener(new FileListener()); try { watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java index 7613e7b3972af..bc157536434f4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java @@ -263,12 +263,10 @@ public void authenticate(final AuthenticationToken authenticationToken, final Ac + tokenPrincipal + "] with header [" + jwtAuthenticationToken.getSignedJWT().getHeader() - + "] and claimSet [" - + jwtAuthenticationToken.getJWTClaimsSet() + "]"; if (logger.isTraceEnabled()) { - logger.trace(msg, ex); + logger.trace(msg + " and claimSet [" + jwtAuthenticationToken.getJWTClaimsSet() + "]", ex); } else { logger.debug(msg + " Cause: " + ex.getMessage()); // only log the stack trace at trace level } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java index 8b3f8ec09675a..0fafd6b63c03f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java @@ -338,7 +338,7 @@ public void cancelled() { } public static Path resolvePath(final Environment environment, final String jwkSetPath) { - final Path directoryPath = environment.configFile(); + final Path directoryPath = environment.configDir(); return directoryPath.resolve(jwkSetPath); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java index d5ef90f7f1664..65e72568cacf8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java @@ -101,7 +101,7 @@ public KerberosRealm(final RealmConfig config, final UserRoleMapper userRoleMapp } this.kerberosTicketValidator = kerberosTicketValidator; this.threadPool = threadPool; - this.keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + this.keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); validateKeytab(this.keytabPath); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java index aa1946f445670..af82b9361dd3d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java @@ -365,7 +365,7 @@ private void validateAccessToken(AccessToken accessToken, JWT idToken) { * @throws IOException if the file cannot be read */ private JWKSet readJwkSetFromFile(String jwkSetPath) throws IOException, ParseException { - final Path path = realmConfig.env().configFile().resolve(jwkSetPath); + final Path path = realmConfig.env().configDir().resolve(jwkSetPath); // avoid using JWKSet.loadFile() as it does not close FileInputStream internally try { String jwkSet = AccessController.doPrivileged( @@ -629,18 +629,20 @@ public void cancelled() { /** * Handle the Token Response from the OpenID Connect Provider. If successful, extract the (yet not validated) Id Token * and access token and call the provided listener. + * (Package private for testing purposes) */ - private static void handleTokenResponse(HttpResponse httpResponse, ActionListener> tokensListener) { + static void handleTokenResponse(HttpResponse httpResponse, ActionListener> tokensListener) { try { final HttpEntity entity = httpResponse.getEntity(); final Header encodingHeader = entity.getContentEncoding(); final Header contentHeader = entity.getContentType(); - if (ContentType.parse(contentHeader.getValue()).getMimeType().equals("application/json") == false) { + final String contentHeaderValue = contentHeader == null ? null : ContentType.parse(contentHeader.getValue()).getMimeType(); + if (contentHeaderValue == null || contentHeaderValue.equals("application/json") == false) { tokensListener.onFailure( new IllegalStateException( "Unable to parse Token Response. Content type was expected to be " + "[application/json] but was [" - + contentHeader.getValue() + + contentHeaderValue + "]" ) ); @@ -688,7 +690,7 @@ private static void handleTokenResponse(HttpResponse httpResponse, ActionListene } catch (Exception e) { tokensListener.onFailure( new ElasticsearchSecurityException( - "Failed to exchange code for Id Token using the Token Endpoint. " + "Unable to parse Token Response", + "Failed to exchange code for Id Token using the Token Endpoint. Unable to parse Token Response", e ) ); @@ -814,7 +816,7 @@ IDTokenValidator createIdTokenValidator(boolean addFileWatcherIfRequired) { } private void setMetadataFileWatcher(String jwkSetPath) throws IOException { - final Path path = realmConfig.env().configFile().resolve(jwkSetPath); + final Path path = realmConfig.env().configDir().resolve(jwkSetPath); FileWatcher watcher = new PrivilegedFileWatcher(path); watcher.addListener(new FileListener(LOGGER, () -> this.idTokenValidator.set(createIdTokenValidator(false)))); watcherService.add(watcher, ResourceWatcherService.Frequency.MEDIUM); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java index 106b550a1e23c..1d69050d7ab25 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java @@ -93,7 +93,7 @@ class SamlMetadataCommand extends KeyStoreAwareCommand { SamlMetadataCommand() { this((environment) -> { - KeyStoreWrapper ksWrapper = KeyStoreWrapper.load(environment.configFile()); + KeyStoreWrapper ksWrapper = KeyStoreWrapper.load(environment.configDir()); return ksWrapper; }); } @@ -458,7 +458,7 @@ private RealmConfig findRealm(Terminal terminal, OptionSet options, Environment final RealmConfig.RealmIdentifier identifier = new RealmConfig.RealmIdentifier(SamlRealmSettings.TYPE, name); final Settings realmSettings = realms.get(identifier); if (realmSettings == null) { - throw new UserException(ExitCodes.CONFIG, "No such realm '" + name + "' defined in " + env.configFile()); + throw new UserException(ExitCodes.CONFIG, "No such realm '" + name + "' defined in " + env.configDir()); } if (isSamlRealm(identifier)) { return buildRealm(identifier, env, settings); @@ -471,10 +471,10 @@ private RealmConfig findRealm(Terminal terminal, OptionSet options, Environment .filter(entry -> isSamlRealm(entry.getKey())) .toList(); if (saml.isEmpty()) { - throw new UserException(ExitCodes.CONFIG, "There is no SAML realm configured in " + env.configFile()); + throw new UserException(ExitCodes.CONFIG, "There is no SAML realm configured in " + env.configDir()); } if (saml.size() > 1) { - terminal.errorPrintln("Using configuration in " + env.configFile()); + terminal.errorPrintln("Using configuration in " + env.configDir()); terminal.errorPrintln( "Found multiple SAML realms: " + saml.stream().map(Map.Entry::getKey).map(Object::toString).collect(Collectors.joining(", ")) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java index 9adfd15e23207..d82be264b2248 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java @@ -792,7 +792,7 @@ private static Tuple names, final ActionListener names, final ActionListener { - QueryBuilder query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE); + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true)); final Supplier supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(SECURITY_ORIGIN)) { SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHash.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHash.java index 8cce453f17fd7..eaaa413f46de6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHash.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHash.java @@ -48,10 +48,10 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings())); try ( SecureString elasticPassword = new SecureString(generatePassword(20)); - KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0])) + KeyStoreWrapper nodeKeystore = KeyStoreWrapper.bootstrap(env.configDir(), () -> new SecureString(new char[0])) ) { nodeKeystore.setString(AUTOCONFIG_ELASTIC_PASSWORD_HASH.getKey(), hasher.hash(elasticPassword)); - nodeKeystore.save(env.configFile(), new char[0]); + nodeKeystore.save(env.configDir(), new char[0]); terminal.print(Terminal.Verbosity.NORMAL, elasticPassword.toString()); } catch (Exception e) { throw new UserException(ExitCodes.CANT_CREATE, "Failed to generate a password for the elastic user", e); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/CreateEnrollmentTokenTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/CreateEnrollmentTokenTool.java index 919f4531734fb..8f5fc96761cc9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/CreateEnrollmentTokenTool.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/tool/CreateEnrollmentTokenTool.java @@ -36,7 +36,7 @@ class CreateEnrollmentTokenTool extends BaseRunAsSuperuserCommand { CreateEnrollmentTokenTool() { this( environment -> new CommandLineHttpClient(environment), - environment -> KeyStoreWrapper.load(environment.configFile()), + environment -> KeyStoreWrapper.load(environment.configDir()), environment -> new ExternalEnrollmentTokenGenerator(environment) ); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java index 61dc638e1d55d..088a9d30513e7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings; import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings; +import org.elasticsearch.xpack.security.PrivilegedFileWatcher; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; import java.io.IOException; @@ -59,7 +60,7 @@ public class FileOperatorUsersStore { public FileOperatorUsersStore(Environment env, ResourceWatcherService watcherService) { this.file = XPackPlugin.resolveConfigFile(env, "operator_users.yml"); this.operatorUsersDescriptor = parseFile(this.file, logger); - FileWatcher watcher = new FileWatcher(file.getParent(), true); + FileWatcher watcher = new PrivilegedFileWatcher(file.getParent(), true); watcher.addListener(new FileOperatorUsersStore.FileListener()); try { watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java index 2f45bafe493bb..542bbbe086cc5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java @@ -93,7 +93,7 @@ public final void execute(Terminal terminal, OptionSet options, Environment env, settingsBuilder.setSecureSettings(keyStoreWrapper); } settings = settingsBuilder.build(); - newEnv = new Environment(settings, env.configFile()); + newEnv = new Environment(settings, env.configDir()); } else { newEnv = env; settings = env.settings(); diff --git a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml index 636627240bf4c..f0992ef48e14c 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,12 +1,60 @@ org.elasticsearch.security: - set_https_connection_properties # for CommandLineHttpClient + - files: + - relative_path: "" + relative_to: config + mode: read + - relative_path: users + relative_to: config + mode: read + exclusive: true + - relative_path: x-pack/users + relative_to: config + mode: read + exclusive: true + - path_setting: xpack.security.authc.realms.ldap.*.files.role_mapping + basedir_if_relative: config + mode: read + exclusive: true + - path_setting: xpack.security.authc.realms.pki.*.files.role_mapping + basedir_if_relative: config + mode: read + exclusive: true + - path_setting: xpack.security.authc.realms.kerberos.*.keytab.path + basedir_if_relative: config + mode: read + exclusive: true + - path_setting: xpack.security.authc.realms.jwt.*.pkc_jwkset_path + basedir_if_relative: config + mode: read + exclusive: true + - path_setting: xpack.security.authc.realms.saml.*.idp.metadata.path + basedir_if_relative: config + mode: read + exclusive: true + io.netty.transport: + - manage_threads - inbound_network - outbound_network io.netty.common: + - manage_threads - inbound_network - outbound_network + - files: + - path: "/etc/os-release" + mode: "read" + - path: "/usr/lib/os-release" + mode: "read" + - path: "/proc/sys/net/core/somaxconn" + mode: read org.opensaml.xmlsec.impl: - write_system_properties: properties: - org.apache.xml.security.ignoreLineBreaks +org.opensaml.saml.impl: + - files: + - path_setting: xpack.security.authc.realms.saml.*.idp.metadata.path + basedir_if_relative: config + mode: read + exclusive: true diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index d814dfbb1c117..b4791207a15bf 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -46,8 +46,9 @@ grant { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; }; grant codeBase "${codebase.netty-transport}" { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentActionTests.java index 0ed6d92fd551d..3ad55d5f64698 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentActionTests.java @@ -69,7 +69,7 @@ public void setup() throws Exception { final Path tempDir = createTempDir(); final Path httpCaPath = tempDir.resolve("httpCa.p12"); Files.copy(getDataPath("/org/elasticsearch/xpack/security/action/enrollment/httpCa.p12"), httpCaPath); - when(env.configFile()).thenReturn(tempDir); + when(env.configDir()).thenReturn(tempDir); final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("keystore.secure_password", "password"); final Settings settings = Settings.builder().put("keystore.path", httpCaPath).setSecureSettings(secureSettings).build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java index 87a20dcd7a12b..9d311dfb90398 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java @@ -78,7 +78,7 @@ public void testDoExecute() throws Exception { Path transportPath = tempDir.resolve("transport.p12"); Files.copy(getDataPath("/org/elasticsearch/xpack/security/action/enrollment/httpCa.p12"), httpCaPath); Files.copy(getDataPath("/org/elasticsearch/xpack/security/action/enrollment/transport.p12"), transportPath); - when(env.configFile()).thenReturn(tempDir); + when(env.configDir()).thenReturn(tempDir); final SSLService sslService = mock(SSLService.class); final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("keystore.secure_password", "password"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesActionTests.java index f1b1f194e5fbf..c1d82505c27fe 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; import org.elasticsearch.xpack.core.security.user.UsernamesField; +import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.junit.BeforeClass; @@ -67,7 +68,12 @@ public void testReservedRoles() { null, Collections.emptySet() ); - TransportGetRolesAction action = new TransportGetRolesAction(mock(ActionFilters.class), rolesStore, transportService); + TransportGetRolesAction action = new TransportGetRolesAction( + mock(ActionFilters.class), + rolesStore, + new ReservedRoleNameChecker.Default(), + transportService + ); final int size = randomIntBetween(1, ReservedRolesStore.names().size()); final List names = randomSubsetOf(size, ReservedRolesStore.names()); @@ -139,7 +145,12 @@ private void testStoreRoles(List storeRoleDescriptors) { null, Collections.emptySet() ); - TransportGetRolesAction action = new TransportGetRolesAction(mock(ActionFilters.class), rolesStore, transportService); + TransportGetRolesAction action = new TransportGetRolesAction( + mock(ActionFilters.class), + rolesStore, + new ReservedRoleNameChecker.Default(), + transportService + ); GetRolesRequest request = new GetRolesRequest(); request.names(storeRoleDescriptors.stream().map(RoleDescriptor::getName).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY)); @@ -200,7 +211,12 @@ public void testGetAllOrMix() { null, Collections.emptySet() ); - TransportGetRolesAction action = new TransportGetRolesAction(mock(ActionFilters.class), rolesStore, transportService); + TransportGetRolesAction action = new TransportGetRolesAction( + mock(ActionFilters.class), + rolesStore, + new ReservedRoleNameChecker.Default(), + transportService + ); final List expectedNames = new ArrayList<>(); if (all) { @@ -275,7 +291,7 @@ public void testGetWithNativeOnly() { requestedNames.addAll(requestedStoreNames); } - final NativeRolesStore rolesStore = mockNativeRolesStore(requestedNames, storeRoleDescriptors); + final NativeRolesStore rolesStore = mockNativeRolesStore(requestedStoreNames, storeRoleDescriptors); final TransportService transportService = new TransportService( Settings.EMPTY, @@ -286,7 +302,12 @@ public void testGetWithNativeOnly() { null, Collections.emptySet() ); - final TransportGetRolesAction action = new TransportGetRolesAction(mock(ActionFilters.class), rolesStore, transportService); + final TransportGetRolesAction action = new TransportGetRolesAction( + mock(ActionFilters.class), + rolesStore, + new ReservedRoleNameChecker.Default(), + transportService + ); final GetRolesRequest request = new GetRolesRequest(); request.names(requestedNames.toArray(Strings.EMPTY_ARRAY)); @@ -298,7 +319,7 @@ public void testGetWithNativeOnly() { verify(rolesStore, times(1)).getRoleDescriptors(eq(new HashSet<>()), anyActionListener()); } else { assertThat(actualRoleNames, containsInAnyOrder(requestedStoreNames.toArray(Strings.EMPTY_ARRAY))); - verify(rolesStore, times(1)).getRoleDescriptors(eq(new HashSet<>(requestedNames)), anyActionListener()); + verify(rolesStore, times(1)).getRoleDescriptors(eq(new HashSet<>(requestedStoreNames)), anyActionListener()); } } @@ -358,7 +379,12 @@ public void testException() { null, Collections.emptySet() ); - TransportGetRolesAction action = new TransportGetRolesAction(mock(ActionFilters.class), rolesStore, transportService); + TransportGetRolesAction action = new TransportGetRolesAction( + mock(ActionFilters.class), + rolesStore, + new ReservedRoleNameChecker.Default(), + transportService + ); GetRolesRequest request = new GetRolesRequest(); request.names(storeRoleDescriptors.stream().map(RoleDescriptor::getName).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java index b84282bd40660..417725d908b41 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java @@ -72,7 +72,7 @@ public void shutdown() { } public void testStore_ConfiguredWithUnreadableFile() throws Exception { - Path configDir = env.configFile(); + Path configDir = env.configDir(); Files.createDirectories(configDir); Path file = configDir.resolve("users"); @@ -88,7 +88,7 @@ public void testStore_ConfiguredWithUnreadableFile() throws Exception { public void testStore_AutoReload() throws Exception { Path users = getDataPath("users"); - Path configDir = env.configFile(); + Path configDir = env.configDir(); Files.createDirectories(configDir); Path file = configDir.resolve("users"); Files.copy(users, file, StandardCopyOption.REPLACE_EXISTING); @@ -149,7 +149,7 @@ private RealmConfig getRealmConfig() { public void testStore_AutoReload_WithParseFailures() throws Exception { Path users = getDataPath("users"); - Path confDir = env.configFile(); + Path confDir = env.configDir(); Files.createDirectories(confDir); Path testUsers = confDir.resolve("users"); Files.copy(users, testUsers, StandardCopyOption.REPLACE_EXISTING); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java index 258770b10c743..759f57a4e0174 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java @@ -285,7 +285,7 @@ private Path writeUsersRoles(String input) throws Exception { } private Path getUsersRolesPath() throws IOException { - Path xpackConf = env.configFile(); + Path xpackConf = env.configDir(); Files.createDirectories(xpackConf); return xpackConf.resolve("users_roles"); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java index 9800cb73faf6a..3d05b7540596a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java @@ -41,7 +41,7 @@ public void testConcurrentReloadWillBeQueuedAndShareTheResults() throws IOExcept final RealmConfig realmConfig = mock(RealmConfig.class); when(realmConfig.getSetting(JwtRealmSettings.PKC_JWKSET_PATH)).thenReturn("jwkset.json"); final Environment env = mock(Environment.class); - when(env.configFile()).thenReturn(tempDir); + when(env.configDir()).thenReturn(tempDir); when(realmConfig.env()).thenReturn(env); final JwkSetLoader jwkSetLoader = spy(new JwkSetLoader(realmConfig, List.of(), null)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java index f1927876eba5f..2c9e57df60e26 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java @@ -59,7 +59,7 @@ public void setup() throws Exception { final RealmConfig realmConfig = mock(RealmConfig.class); when(realmConfig.getSetting(JwtRealmSettings.PKC_JWKSET_PATH)).thenReturn("jwkset.json"); final Environment env = mock(Environment.class); - when(env.configFile()).thenReturn(tempDir); + when(env.configDir()).thenReturn(tempDir); when(realmConfig.env()).thenReturn(env); validateSignatureAttemptCounter = new AtomicInteger(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java index f01914a7fed06..b15edd943db52 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java @@ -63,7 +63,7 @@ public void testAuthenticateDifferentFailureScenarios() throws LoginException, G final boolean throwExceptionForInvalidTicket = validTicket ? false : randomBoolean(); final boolean throwLoginException = randomBoolean(); final byte[] decodedTicket = randomByteArrayOfLength(5); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); if (validTicket) { mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, outToken), null); @@ -144,7 +144,7 @@ public void testDelegatedAuthorizationFailedToResolve() throws Exception { settings = Settings.builder().put(settings).putList("authorization_realms", "other_realm").build(); final KerberosRealm kerberosRealm = createKerberosRealm(Collections.singletonList(otherRealm), username); final byte[] decodedTicket = "base64encodedticket".getBytes(StandardCharsets.UTF_8); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, "out-token"), null); final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java index b1ddb631a8dd2..c6431a8d81685 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java @@ -48,7 +48,7 @@ public void testAuthenticateWithCache() throws LoginException, GSSException { metadata.put(KerberosRealm.KRB_METADATA_UPN_KEY, username); final User expectedUser = new User(expectedUsername, roles.toArray(new String[0]), null, null, metadata, true); final byte[] decodedTicket = randomByteArrayOfLength(10); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, outToken), null); final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket); @@ -78,7 +78,7 @@ public void testCacheInvalidationScenarios() throws LoginException, GSSException final String authNUsername = randomFrom(userNames); final byte[] decodedTicket = randomByteArrayOfLength(10); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(authNUsername, outToken), null); final String expectedUsername = maybeRemoveRealmName(authNUsername); @@ -137,7 +137,7 @@ public void testAuthenticateWithValidTicketSucessAuthnWithUserDetailsWhenCacheDi metadata.put(KerberosRealm.KRB_METADATA_UPN_KEY, username); final User expectedUser = new User(expectedUsername, roles.toArray(new String[0]), null, null, metadata, true); final byte[] decodedTicket = randomByteArrayOfLength(10); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, outToken), null); final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java index 177507ce6d792..e4718f3e95019 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java @@ -82,7 +82,7 @@ public void testAuthenticateWithValidTicketSucessAuthnWithUserDetails() throws L metadata.put(KerberosRealm.KRB_METADATA_UPN_KEY, username); final User expectedUser = new User(expectedUsername, roles.toArray(new String[roles.size()]), null, null, metadata, true); final byte[] decodedTicket = "base64encodedticket".getBytes(StandardCharsets.UTF_8); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, "out-token"), null); final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket); @@ -106,7 +106,7 @@ public void testFailedAuthorization() throws LoginException, GSSException { final String username = randomPrincipalName(); final KerberosRealm kerberosRealm = createKerberosRealm(username); final byte[] decodedTicket = "base64encodedticket".getBytes(StandardCharsets.UTF_8); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>("does-not-exist@REALM", "out-token"), null); @@ -236,7 +236,7 @@ public void testDelegatedAuthorization() throws Exception { final KerberosRealm kerberosRealm = createKerberosRealm(Collections.singletonList(otherRealm), username); final User expectedUser = lookupUser; final byte[] decodedTicket = "base64encodedticket".getBytes(StandardCharsets.UTF_8); - final Path keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); + final Path keytabPath = config.env().configDir().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); final boolean krbDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, "out-token"), null); final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java index f839e5e7c1dcb..76069ce500ad9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java @@ -968,6 +968,23 @@ public void testHandleUserinfoResponseFailure() throws Exception { ); } + public void testHandleTokenResponseNullContentType() { + final HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, RestStatus.OK.getStatus(), ""); + final StringEntity entity = new StringEntity("", (ContentType) null); + response.setEntity(entity); + + final PlainActionFuture> future = new PlainActionFuture<>(); + OpenIdConnectAuthenticator.handleTokenResponse(response, future); + final IllegalStateException exception = expectThrows(IllegalStateException.class, future::actionGet); + + assertThat( + exception, + TestMatchers.throwableWithMessage( + "Unable to parse Token Response. Content type was expected to be [application/json] but was [null]" + ) + ); + } + public void testLogIdTokenAndNonce() throws URISyntaxException, BadJOSEException, JOSEException, IllegalAccessException { final Logger logger = LogManager.getLogger(OpenIdConnectAuthenticator.class); Loggers.setLevel(logger, Level.DEBUG); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/FileServiceAccountTokenStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/FileServiceAccountTokenStoreTests.java index 00b55e5b48337..0f2a720660afd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/FileServiceAccountTokenStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/FileServiceAccountTokenStoreTests.java @@ -128,7 +128,7 @@ public void testParseFileNotExists() throws IllegalAccessException, IOException public void testAutoReload() throws Exception { Path serviceTokensSourceFile = getDataPath("service_tokens"); - Path configDir = env.configFile(); + Path configDir = env.configDir(); Files.createDirectories(configDir); Path targetFile = configDir.resolve("service_tokens"); Files.copy(serviceTokensSourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING); @@ -225,7 +225,7 @@ public void testAutoReload() throws Exception { public void testFindTokensFor() throws IOException { Path serviceTokensSourceFile = getDataPath("service_tokens"); - Path configDir = env.configFile(); + Path configDir = env.configDir(); Files.createDirectories(configDir); Path targetFile = configDir.resolve("service_tokens"); Files.copy(serviceTokensSourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java index 6332e63ca5958..ee025fe64ff9a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapperTests.java @@ -76,8 +76,8 @@ public class DnRoleMapperTests extends ESTestCase { public void init() throws IOException { settings = Settings.builder().put("resource.reload.interval.high", "100ms").put("path.home", createTempDir()).build(); env = TestEnvironment.newEnvironment(settings); - if (Files.exists(env.configFile()) == false) { - Files.createDirectory(env.configFile()); + if (Files.exists(env.configDir()) == false) { + Files.createDirectory(env.configDir()); } threadPool = new TestThreadPool("test"); } @@ -100,7 +100,7 @@ public void testMapper_ConfiguredWithUnreadableFile() throws Exception { public void testMapper_AutoReload() throws Exception { Path roleMappingFile = getDataPath("role_mapping.yml"); - Path file = env.configFile().resolve("test_role_mapping.yml"); + Path file = env.configDir().resolve("test_role_mapping.yml"); Files.copy(roleMappingFile, file, StandardCopyOption.REPLACE_EXISTING); final CountDownLatch latch = new CountDownLatch(1); @@ -144,7 +144,7 @@ public void testMapper_AutoReload() throws Exception { public void testMapper_AutoReload_WithParseFailures() throws Exception { Path roleMappingFile = getDataPath("role_mapping.yml"); - Path file = env.configFile().resolve("test_role_mapping.yml"); + Path file = env.configDir().resolve("test_role_mapping.yml"); Files.copy(roleMappingFile, file, StandardCopyOption.REPLACE_EXISTING); final CountDownLatch latch = new CountDownLatch(1); @@ -171,7 +171,7 @@ public void testMapper_AutoReload_WithParseFailures() throws Exception { public void testMapperAutoReloadWithoutListener() throws Exception { Path roleMappingFile = getDataPath("role_mapping.yml"); - Path file = env.configFile().resolve("test_role_mapping.yml"); + Path file = env.configDir().resolve("test_role_mapping.yml"); Files.copy(roleMappingFile, file, StandardCopyOption.REPLACE_EXISTING); try (ResourceWatcherService watcherService = new ResourceWatcherService(settings, threadPool)) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index a8bb537878d5a..a1c1bfc6b6b71 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -465,22 +465,6 @@ public void testSelectorsDoNotImpactWildcardDetection() { ); } - public void testWildcardSelectorsAreNotAllowedInShardLevelRequests() { - ShardSearchRequest request = mock(ShardSearchRequest.class); - when(request.indices()).thenReturn(new String[] { "index10::*" }); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> defaultIndicesResolver.resolveIndicesAndAliasesWithoutWildcards(TransportSearchAction.TYPE.name() + "[s]", request) - ); - assertThat( - exception, - throwableWithMessage( - "the action indices:data/read/search[s] does not support wildcard selectors;" - + " the provided index expression(s) [index10::*] are not allowed" - ) - ); - } - public void testAllIsNotAllowedInShardLevelRequests() { ShardSearchRequest request = mock(ShardSearchRequest.class); final boolean literalAll = randomBoolean(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStoreTests.java index 34cfde8dc862f..972c00b59b1f2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStoreTests.java @@ -484,7 +484,7 @@ public void testParseInvalidConfig() throws IOException { } private Path getOperatorUsersPath() throws IOException { - Path xpackConf = env.configFile(); + Path xpackConf = env.configDir(); Files.createDirectories(xpackConf); return xpackConf.resolve("operator_users.yml"); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java index 00f170a4cf8d8..3a3ae55cad090 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java @@ -1130,7 +1130,7 @@ public void testProfileFiltersCreatedDifferentlyForDifferentTransportAndRemoteCl mock(SslKeyConfig.class), randomFrom(SslVerificationMode.values()), SslClientAuthenticationMode.NONE, - List.of("TLS_RSA_WITH_AES_256_GCM_SHA384"), + List.of(Runtime.version().feature() < 24 ? "TLS_RSA_WITH_AES_256_GCM_SHA384" : "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"), List.of("TLSv1.2") ) ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java index 9bb0c8af6f481..4f64b780e1f97 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.jdk.RuntimeVersionFeature; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ssl.SSLService; import org.junit.Before; @@ -362,6 +363,11 @@ private void checkBlockedResource( String configKey, BiConsumer configure ) throws Exception { + assumeTrue( + "Requires Security Manager to block access, entitlements are not checked for unit tests", + RuntimeVersionFeature.isSecurityManagerAvailable() + ); + final String prefix = randomSslPrefix(); final Settings.Builder settings = Settings.builder(); configure.accept(prefix, settings); @@ -375,7 +381,7 @@ private void checkBlockedResource( + " [" + fileName + "] because access to read the file is blocked; SSL resources should be placed in the [" - + env.configFile().toAbsolutePath().toString() + + env.configDir().toAbsolutePath().toString() + "] directory"; Throwable exception = expectFailure(settings); @@ -477,7 +483,7 @@ private Settings.Builder configureWorkingKeystore(String prefix, Settings.Builde private ElasticsearchException expectFailure(Settings.Builder settings) { return expectThrows( ElasticsearchException.class, - () -> new SSLService(new Environment(buildEnvSettings(settings.build()), env.configFile())) + () -> new SSLService(new Environment(buildEnvSettings(settings.build()), env.configDir())) ); } diff --git a/x-pack/plugin/slm/build.gradle b/x-pack/plugin/slm/build.gradle index 332941215abc8..4049cb9b80d6e 100644 --- a/x-pack/plugin/slm/build.gradle +++ b/x-pack/plugin/slm/build.gradle @@ -2,15 +2,18 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name = 'x-pack-slm' description = 'Elasticsearch Expanded Pack Plugin - Snapshot Lifecycle Management' - classname ='org.elasticsearch.xpack.slm.SnapshotLifecycle' + classname = 'org.elasticsearch.xpack.slm.SnapshotLifecycle' extendedPlugins = ['x-pack-core'] - hasNativeController =false - requiresKeystore =true + hasNativeController = false + requiresKeystore = true } + base { archivesName = 'x-pack-slm' } @@ -21,6 +24,20 @@ dependencies { testImplementation project(xpackModule('ccr')) testImplementation project(xpackModule('ilm')) testImplementation project(':modules:data-streams') + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation project(xpackModule('slm')) + yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) + clusterModules project(xpackModule("ilm")) + clusterModules project(xpackModule("searchable-snapshots")) + clusterModules project(":modules:data-streams") +} + +restResources { + restApi { + include '_common', 'cluster', 'indices', 'index', 'snapshot', 'slm', 'health_report' + } } -addQaCheckDependencies(project) +tasks.named("javaRestTest") { + usesDefaultDistribution("uses _xpack/usage api") +} diff --git a/x-pack/plugin/slm/qa/build.gradle b/x-pack/plugin/slm/qa/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/slm/qa/multi-node/build.gradle b/x-pack/plugin/slm/qa/multi-node/build.gradle deleted file mode 100644 index d6b1fe8a1e219..0000000000000 --- a/x-pack/plugin/slm/qa/multi-node/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -import org.elasticsearch.gradle.internal.info.BuildParams -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.legacy-java-rest-test' - -dependencies { - javaRestTestImplementation(testArtifact(project(xpackModule('core')))) - javaRestTestImplementation project(xpackModule('slm')) -} - -File repoDir = file("$buildDir/testclusters/repo") - -tasks.named("javaRestTest").configure { - /* To support taking index snapshots, we have to set path.repo setting */ - nonInputProperties.systemProperty 'tests.path.repo', repoDir -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - numberOfNodes = 4 - - setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE - setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' - setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' - setting 'xpack.security.enabled', 'false' - setting 'xpack.watcher.enabled', 'false' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' - setting 'indices.lifecycle.poll_interval', '1000ms' - setting 'logger.org.elasticsearch.xpack.core.slm', 'TRACE' - setting 'logger.org.elasticsearch.xpack.slm', 'TRACE' -} - -if (buildParams.inFipsJvm){ - // Test clusters run with security disabled - tasks.named("javaRestTest").configure{enabled = false } -} diff --git a/x-pack/plugin/slm/qa/rest/build.gradle b/x-pack/plugin/slm/qa/rest/build.gradle deleted file mode 100644 index 3e23d52a291a9..0000000000000 --- a/x-pack/plugin/slm/qa/rest/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' -apply plugin: 'elasticsearch.authenticated-testclusters' - -dependencies { - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -restResources { - restApi { - include '_common', 'cluster', 'indices', 'index', 'snapshot', 'slm', 'health_report' - } -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.autoconfiguration.enabled', 'false' -} diff --git a/x-pack/plugin/slm/qa/with-security/build.gradle b/x-pack/plugin/slm/qa/with-security/build.gradle deleted file mode 100644 index d99d050254c2c..0000000000000 --- a/x-pack/plugin/slm/qa/with-security/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -apply plugin: 'elasticsearch.internal-java-rest-test' - -dependencies { - javaRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -tasks.named("javaRestTest").configure { - usesDefaultDistribution() -} diff --git a/x-pack/plugin/slm/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/slm/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java similarity index 100% rename from x-pack/plugin/slm/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java rename to x-pack/plugin/slm/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java diff --git a/x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/slm/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java similarity index 96% rename from x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java rename to x-pack/plugin/slm/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java index 6ee35086baf76..52a01c3bff12b 100644 --- a/x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java +++ b/x-pack/plugin/slm/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java @@ -22,7 +22,10 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.ToXContent; @@ -35,6 +38,9 @@ import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats; import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.io.InputStream; @@ -62,9 +68,31 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; +@SuppressForbidden(reason = "TemporaryFolder uses java.io.File") public class SnapshotLifecycleRestIT extends ESRestTestCase { private static final String NEVER_EXECUTE_CRON_SCHEDULE = "* * * 31 FEB ? *"; + public static TemporaryFolder repoDir = new TemporaryFolder(); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + // TODO: Revert to integ-test distro once we sort out issues with usage and info xpack apis + .distribution(DistributionType.DEFAULT) + .nodes(2) + .module("x-pack-slm") + .module("x-pack-ilm") + .module("searchable-snapshots") + .module("data-streams") + .setting("path.repo", () -> repoDir.getRoot().getAbsolutePath()) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") + .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") + .setting("indices.lifecycle.poll_interval", "1000ms") + .build(); + + @ClassRule + public static RuleChain rules = RuleChain.outerRule(repoDir).around(cluster); + // as we are testing the SLM history entries we'll preserve the "slm-history-ilm-policy" policy as it'll be associated with the // .slm-history-* indices and we won't be able to delete it when we wipe out the cluster @Override @@ -72,6 +100,11 @@ protected boolean preserveILMPoliciesUponCompletion() { return true; } + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + public void testMissingRepo() throws Exception { SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( "missing-repo-policy", @@ -919,7 +952,7 @@ private void initializeRepo(String repoName, String maxBytesPerSecond) throws IO .field("type", "fs") .startObject("settings") .field("compress", randomBoolean()) - .field("location", System.getProperty("tests.path.repo")) + .field("location", repoDir.getRoot().getAbsolutePath()) .field("max_snapshot_bytes_per_sec", maxBytesPerSecond) .endObject() .endObject() diff --git a/x-pack/plugin/slm/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleYamlIT.java b/x-pack/plugin/slm/src/yamlRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleYamlIT.java similarity index 72% rename from x-pack/plugin/slm/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleYamlIT.java rename to x-pack/plugin/slm/src/yamlRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleYamlIT.java index f3f542cde3365..d8f243121939f 100644 --- a/x-pack/plugin/slm/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleYamlIT.java +++ b/x-pack/plugin/slm/src/yamlRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleYamlIT.java @@ -15,16 +15,27 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; import java.util.Objects; @TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs public class SnapshotLifecycleYamlIT extends ESClientYamlSuiteTestCase { - private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username")); - private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password")); + private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username", "test_admin")); + private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password", "x-pack-test-password")); + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("x-pack-slm") + .module("x-pack-ilm") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user(USER, PASS) + .build(); public SnapshotLifecycleYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); @@ -40,4 +51,9 @@ protected Settings restClientSettings() { String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml b/x-pack/plugin/slm/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml similarity index 100% rename from x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml rename to x-pack/plugin/slm/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml diff --git a/x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/20_health.yml b/x-pack/plugin/slm/src/yamlRestTest/resources/rest-api-spec/test/slm/20_health.yml similarity index 100% rename from x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/20_health.yml rename to x-pack/plugin/slm/src/yamlRestTest/resources/rest-api-spec/test/slm/20_health.yml diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 1f55ee3c25c52..1e7b21b6294a9 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -28,7 +28,7 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'false' } -if (buildParams.isSnapshotBuild() == false) { +if (buildParams.snapshotBuild == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 67d25556a2aa7..f7c5f1b8072f3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.GeoShapeIndexer; @@ -300,14 +301,17 @@ protected Function, List> getFormatter(String format) { } @Override - protected boolean isBoundsExtractionSupported() { - // Extracting bounds for geo shapes is not implemented yet. - return false; + public BlockLoader blockLoader(BlockLoaderContext blContext) { + return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS + ? new GeoBoundsBlockLoader(name()) + : blockLoaderFromSource(blContext); } - @Override - protected CoordinateEncoder coordinateEncoder() { - return CoordinateEncoder.GEO; + static class GeoBoundsBlockLoader extends AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader { + + GeoBoundsBlockLoader(String fieldName) { + super(fieldName); + } } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 2d586ac8eb86a..c5f5f64d3e3d8 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapper; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -31,6 +32,7 @@ import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField; import org.elasticsearch.lucene.spatial.CartesianShapeIndexer; import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.lucene.spatial.Extent; import org.elasticsearch.lucene.spatial.XYQueriesUtils; import org.elasticsearch.script.field.AbstractScriptFieldFactory; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -186,13 +188,26 @@ protected Function, List> getFormatter(String format) { } @Override - protected boolean isBoundsExtractionSupported() { - return true; + public BlockLoader blockLoader(BlockLoaderContext blContext) { + return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS + ? new CartesianBoundsBlockLoader(name()) + : blockLoaderFromSource(blContext); } - @Override - protected CoordinateEncoder coordinateEncoder() { - return CoordinateEncoder.CARTESIAN; + static class CartesianBoundsBlockLoader extends BoundsBlockLoader { + protected CartesianBoundsBlockLoader(String fieldName) { + super(fieldName); + } + + protected void writeExtent(BlockLoader.IntBuilder builder, Extent extent) { + // For cartesian_shape we store 4 values as a multi-valued field, in the same order as the fields in the Rectangle class + builder.beginPositionEntry(); + builder.appendInt(Math.min(extent.negLeft, extent.posLeft)); + builder.appendInt(Math.max(extent.negRight, extent.posRight)); + builder.appendInt(extent.top); + builder.appendInt(extent.bottom); + builder.endPositionEntry(); + } } } diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle index d1b179f09e403..fd0e43fb23c3c 100644 --- a/x-pack/plugin/sql/jdbc/build.gradle +++ b/x-pack/plugin/sql/jdbc/build.gradle @@ -20,11 +20,20 @@ dependencies { testImplementation project(':modules:rest-root') } -tasks.named("compileJava").configure { +java { targetCompatibility = JavaVersion.VERSION_1_8 sourceCompatibility = JavaVersion.VERSION_1_8 } +tasks.named("compileTestJava").configure { + targetCompatibility = buildParams.getMinimumRuntimeVersion() + sourceCompatibility = buildParams.getMinimumRuntimeVersion() +} + +tasks.named("test").configure { + // reset the unit test classpath as using the shadow jar won't work due to relocated packages + classpath = sourceSets.test.runtimeClasspath +} tasks.named("shadowJar").configure { relocate 'com.fasterxml', 'shadow.fasterxml' @@ -34,7 +43,3 @@ tasks.named("shadowJar").configure { } } -tasks.named("test").configure { - // reset the unit test classpath as using the shadow jar won't work due to relocated packages - classpath = sourceSets.test.runtimeClasspath -} diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec index 2fa82c05cc1aa..0bdd3fbc1b450 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec @@ -3353,7 +3353,7 @@ Alejandro Amabile Anoosh Basil -Brendon +Cristinel // end::filterToday ; diff --git a/x-pack/plugin/sql/sql-client/build.gradle b/x-pack/plugin/sql/sql-client/build.gradle index 4a20e00666ea4..f52e4b9ed96db 100644 --- a/x-pack/plugin/sql/sql-client/build.gradle +++ b/x-pack/plugin/sql/sql-client/build.gradle @@ -12,11 +12,16 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) } -tasks.named("compileJava").configure { +java { targetCompatibility = JavaVersion.VERSION_1_8 sourceCompatibility = JavaVersion.VERSION_1_8 } +tasks.named("compileTestJava").configure { + targetCompatibility = buildParams.getMinimumRuntimeVersion() + sourceCompatibility = buildParams.getMinimumRuntimeVersion() +} + tasks.named('forbiddenApisMain').configure { // does not depend on core, so only jdk and http signatures should be checked replaceSignatureFiles 'jdk-signatures' diff --git a/x-pack/plugin/sql/sql-proto/build.gradle b/x-pack/plugin/sql/sql-proto/build.gradle index 2cb1cfa89f033..0a34afdbc2504 100644 --- a/x-pack/plugin/sql/sql-proto/build.gradle +++ b/x-pack/plugin/sql/sql-proto/build.gradle @@ -16,11 +16,16 @@ dependencies { } } -tasks.named("compileJava").configure { +java { targetCompatibility = JavaVersion.VERSION_1_8 sourceCompatibility = JavaVersion.VERSION_1_8 } +tasks.named("compileTestJava").configure { + targetCompatibility = buildParams.getMinimumRuntimeVersion() + sourceCompatibility = buildParams.getMinimumRuntimeVersion() +} + tasks.named('forbiddenApisMain').configure { //sql does not depend on server, so only jdk signatures should be checked replaceSignatureFiles 'jdk-signatures' diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/deprecation/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/deprecation/10_basic.yml index 78015468ba673..92a48e737d453 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/deprecation/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/deprecation/10_basic.yml @@ -11,9 +11,11 @@ setup: - method: GET path: /_migration/deprecations capabilities: [ data_streams, ilm_policies, templates ] - test_runner_features: capabilities + test_runner_features: [capabilities, allowed_warnings] reason: "Support for data streams, ILM policies and templates" - do: + allowed_warnings: + - "this request accesses system indices: [.security-7], but in a future major version, direct access to system indices will be prevented by default" migration.deprecations: index: "*" - length: { cluster_settings: 0 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml index cffc161b11539..666d7939c04bf 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml @@ -338,3 +338,58 @@ esql.query: body: query: 'FROM test_grok | KEEP name | WHERE last_name == "Facello" | EVAL name = concat("1 ", last_name) | GROK name "%{NUMBER:foo} %{WORD:foo}"' +--- +"union types with null blocks from missing fields #125850": + - requires: + test_runner_features: [allowed_warnings_regex, capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [fix_doubly_released_null_blocks_in_valuesourcereader] + reason: "fixed handing out already closed null block references in ValueSourceReader" + - do: + indices.create: + index: test1 + body: + mappings: + properties: + truefalse1 : + type : boolean + truefalse2 : + type: boolean + - do: + indices.create: + index: test2 + body: + mappings: + properties: + truefalse1 : + type : keyword + truefalse2 : + type: keyword + - do: + bulk: + refresh: true + body: + - { "index": { "_index": "test1" } } + - { "truefalse1": null} + - { "index": { "_index": "test2" } } + - { "truefalse1": null } + + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + + esql.query: + body: + query: 'FROM test* | eval t1 = truefalse1::boolean, t2 = truefalse2::boolean | keep t1, t2' + - match: { columns.0.name: t1 } + - match: { columns.0.type: boolean } + - match: { columns.1.name: t2 } + - match: { columns.1.type: boolean } + - length: { values: 2 } + - match: { values.0.0: null } + - match: { values.0.1: null } + - match: { values.1.0: null } + - match: { values.1.1: null } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml index b3de28b70162e..62a49422079b8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml @@ -25,17 +25,3 @@ } } - match: { error.reason: "Unknown task_type [bad]" } - ---- -"Test inference with bad task type": - - do: - catch: bad_request - inference.inference: - task_type: bad - inference_id: elser_model - body: > - { - "input": "important text" - } - - match: { error.reason: "Unknown task_type [bad]" } - diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml similarity index 100% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml similarity index 56% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml index b1d6b0630d0a7..0f29389953c38 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml @@ -1,11 +1,10 @@ -"Get feature upgrade status": +"Start feature upgrade": - requires: cluster_features: ["gte_v7.16.0"] reason: "Endpoint added in 7.16.0" - do: - migration.get_feature_upgrade_status: {} + migration.post_feature_upgrade: {} - is_false: accepted - - is_true: features diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/learning_to_rank_rescorer.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/learning_to_rank_rescorer.yml deleted file mode 100644 index 5c0096e9666fc..0000000000000 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/learning_to_rank_rescorer.yml +++ /dev/null @@ -1,325 +0,0 @@ -setup: - - skip: - features: headers - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - ml.put_trained_model: - model_id: ltr-model - body: > - { - "description": "super complex model for tests", - "inference_config": { - "learning_to_rank": { - "feature_extractors": [ - { - "query_extractor": { - "feature_name": "cost", - "query": {"script_score": {"query": {"match_all":{}}, "script": {"source": "return doc['cost'].value;"}}} - } - }, - { - "query_extractor": { - "feature_name": "type_tv", - "query": {"term": {"product": "TV"}} - } - }, - { - "query_extractor": { - "feature_name": "type_vcr", - "query": {"term": {"product": "VCR"}} - } - }, - { - "query_extractor": { - "feature_name": "type_laptop", - "query": {"term": {"product": "Laptop"}} - } - } - ] - } - }, - "definition": { - "trained_model": { - "ensemble": { - "feature_names": ["cost", "type_tv", "type_vcr", "type_laptop"], - "target_type": "regression", - "trained_models": [ - { - "tree": { - "feature_names": [ - "cost" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 400, - "decision_type": "lte", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 5.0 - }, - { - "node_index": 2, - "leaf_value": 2.0 - } - ], - "target_type": "regression" - } - }, - { - "tree": { - "feature_names": [ - "type_tv" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 1, - "decision_type": "lt", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 1.0 - }, - { - "node_index": 2, - "leaf_value": 12.0 - } - ], - "target_type": "regression" - } - } - ] - } - } - } - } - - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - indices.create: - index: store - body: - mappings: - properties: - product: - type: keyword - cost: - type: integer - - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - Content-Type: application/json - bulk: - index: store - refresh: true - body: | - { "index": {} } - { "product": "TV", "cost": 300 } - { "index": {} } - { "product": "TV", "cost": 400} - { "index": {} } - { "product": "TV", "cost": 600} - { "index": {} } - { "product": "VCR", "cost": 15} - { "index": {} } - { "product": "VCR", "cost": 350} - { "index": {} } - { "product": "VCR", "cost": 580} - { "index": {} } - { "product": "Laptop", "cost": 100} - { "index": {} } - { "product": "Laptop", "cost": 300} - { "index": {} } - { "product": "Laptop", "cost": 500} - ---- -"Test rescore with stored model": - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/80703" - - - do: - search: - index: store - size: 3 - body: > - { - "rescore": { - "window_size": 10, - "learning_to_rank": { "model_id": "ltr-model" } - } - } - - match: { hits.hits.0._score: 17.0 } - - match: { hits.hits.1._score: 17.0 } - - match: { hits.hits.2._score: 14.0 } - - - do: - search: - index: store - size: 3 - body: > - { - "query": {"term": {"product": "Laptop"}}, - "rescore": { - "window_size": 10, - "learning_to_rank": { "model_id": "ltr-model" } - } - } - - match: { hits.hits.0._score: 6.0 } - - match: { hits.hits.1._score: 6.0 } - - match: { hits.hits.2._score: 3.0 } ---- -"Test rescore with stored model and smaller window_size": - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/80703" - - - do: - search: - index: store - size: 5 - body: > - { - "rescore": { - "window_size": 2, - "learning_to_rank": { "model_id": "ltr-model" } - } - } - - match: { hits.hits.0._score: 17.0 } - - match: { hits.hits.1._score: 17.0 } - - match: { hits.hits.2._score: 1.0 } - - match: { hits.hits.3._score: 1.0 } - - match: { hits.hits.4._score: 1.0 } ---- -"Test rescore with stored model and chained rescorers": - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/80703" - - - do: - search: - index: store - size: 5 - body: > - { - "rescore": [ - { - "window_size": 4, - "query": { "rescore_query":{ "script_score": {"query": {"match_all": {}}, "script": {"source": "return 4"}}}} - }, - { - "window_size": 3, - "learning_to_rank": { "model_id": "ltr-model" } - }, - { - "window_size": 2, - "query": { "rescore_query": { "script_score": {"query": {"match_all": {}}, "script": {"source": "return 20"}}}} - } - ] - } - - match: { hits.hits.0._score: 37.0 } - - match: { hits.hits.1._score: 37.0 } - - match: { hits.hits.2._score: 14.0 } - - match: { hits.hits.3._score: 5.0 } - - match: { hits.hits.4._score: 1.0 } ---- -"Test rescore with missing model": - - do: - catch: missing - search: - index: store - body: > - { - "rescore": { - "window_size": 10, - "learning_to_rank": { "model_id": "ltr-missing" } - } - } ---- -"Test rescore with no hits model": - - do: - search: - index: store - body: > - { - "query": {"term": {"product": "Speaker"}}, - "rescore": { - "window_size": 10, - "learning_to_rank": { "model_id": "ltr-model" } - } - } - - length: { hits.hits: 0 } ---- -"Test model input validation": - - skip: - features: headers - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - catch: bad_request - ml.put_trained_model: - model_id: bad-model - body: > - { - "description": "a bad model", - "input": { - "field_names": ["cost"] - }, - "inference_config": { - "learning_to_rank": { } - }, - "definition": { - "trained_model": { - "ensemble": { - "feature_names": ["cost"], - "target_type": "regression", - "trained_models": [ - { - "tree": { - "feature_names": [ - "cost" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 400, - "decision_type": "lte", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 5.0 - }, - { - "node_index": 2, - "leaf_value": 2.0 - } - ], - "target_type": "regression" - } - } - ] - } - } - } - } - - - match: { status: 400 } - - match: { error.root_cause.0.type: "action_request_validation_exception" } - - match: { error.root_cause.0.reason: "Validation Failed: 1: cannot specify [input.field_names] for a model of type [learning_to_rank];" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml index 04c7c0011e6b2..1d6837b462328 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml @@ -1,11 +1,11 @@ --- "Bulk indexing of monitoring data": - skip: - features: ["allowed_warnings"] + features: ["allowed_warnings_regex"] - do: - allowed_warnings: - - "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release." + allowed_warnings_regex: + - "\\[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.*" cluster.put_settings: body: persistent: @@ -172,11 +172,11 @@ --- "Bulk indexing of monitoring data on closed indices should throw an export exception": - skip: - features: ["allowed_warnings"] + features: ["allowed_warnings", "allowed_warnings_regex"] - do: - allowed_warnings: - - "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release." + allowed_warnings_regex: + - "\\[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.*" cluster.put_settings: body: persistent: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml index 93e5c44b83e2a..7949a27cc9eac 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml @@ -76,11 +76,11 @@ teardown: --- "Monitoring Bulk API": - skip: - features: ["catch_unauthorized", "allowed_warnings"] + features: ["catch_unauthorized", "allowed_warnings_regex"] - do: - allowed_warnings: - - "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release." + allowed_warnings_regex: + - "\\[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.*" cluster.put_settings: body: persistent: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml index 791712ee925a5..837afecb3625b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml @@ -135,3 +135,28 @@ setup: id: "1" body: vector1: [[2, -1, 1], [[2, -1, 1]]] +--- +"Updating dim to null is not allowed": + - requires: + cluster_features: "mapper.npe_on_dims_update_fix" + reason: "dims update fix" + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: rank_vectors + dims: 4 + - do: + catch: bad_request + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: rank_vectors diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml index 9ba0b5e4088af..5e167e5a4dded 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml @@ -115,7 +115,10 @@ setup: snapshot: test_snapshot_2 wait_for_completion: true body: | - { "indices": "test_synthetic" } + { + "indices": "test_synthetic", + "include_global_state": false + } - match: { snapshot.snapshot: test_snapshot_2 } - match: { snapshot.state : PARTIAL } @@ -132,7 +135,10 @@ setup: snapshot: test_snapshot_3 wait_for_completion: true body: | - { "indices": "test_*" } + { + "indices": "test_*", + "include_global_state": false + } - match: { snapshot.snapshot: test_snapshot_3 } - match: { snapshot.state : PARTIAL } diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index 1796ba334cdaf..1d8464c45edc8 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -1,13 +1,15 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { name = 'x-pack-stack' description = 'Elasticsearch Expanded Pack Plugin - Stack' - classname ='org.elasticsearch.xpack.stack.StackPlugin' + classname = 'org.elasticsearch.xpack.stack.StackPlugin' extendedPlugins = ['x-pack-core'] - hasNativeController =false - requiresKeystore =true + hasNativeController = false + requiresKeystore = true } base { @@ -19,16 +21,30 @@ dependencies { testImplementation project(':modules:data-streams') javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation project(path: ':x-pack:plugin:stack') + yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) clusterModules project(':modules:mapper-extras') clusterModules project(xpackModule('wildcard')) + clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-constant-keyword')) + clusterModules project(':modules:ingest-common') + clusterModules project(':modules:mapper-extras') + clusterModules project(':modules:data-streams') +} + +restResources { + restApi { + include '_common', 'cluster', 'indices', 'index', 'snapshot', 'ilm', 'slm', 'stack', 'indices' + } } // These tests are only invoked direclty as part of a dedicated build job -tasks.named('javaRestTest').configure {task -> - onlyIf("E2E test task must be invoked directly") { - gradle.startParameter.getTaskNames().contains(task.path) || - (gradle.startParameter.getTaskNames().contains(task.name) && gradle.startParameter.currentDir == project.projectDir) - } +tasks.named('javaRestTest') { task -> + onlyIf("E2E test task must be invoked directly") { + gradle.startParameter.getTaskNames().contains(task.path) || + (gradle.startParameter.getTaskNames().contains(task.name) && gradle.startParameter.currentDir == project.projectDir) + } } -addQaCheckDependencies(project) +tasks.named("yamlRestTestV7CompatTransform").configure({ task -> + task.skipTest("stack/10_basic/Test kibana reporting index auto creation", "warning does not exist for compatibility") +}) diff --git a/x-pack/plugin/stack/qa/build.gradle b/x-pack/plugin/stack/qa/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle deleted file mode 100644 index 09ff872df3584..0000000000000 --- a/x-pack/plugin/stack/qa/rest/build.gradle +++ /dev/null @@ -1,23 +0,0 @@ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' -apply plugin: 'elasticsearch.authenticated-testclusters' - -dependencies { - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) -} - -restResources { - restApi { - include '_common', 'cluster', 'indices', 'index', 'snapshot', 'ilm', 'slm', 'stack', 'indices' - } -} - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' -} - -tasks.named("yamlRestTestV7CompatTransform").configure({ task -> - task.skipTest("stack/10_basic/Test kibana reporting index auto creation", "warning does not exist for compatibility") -}) diff --git a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/stack/StackYamlIT.java b/x-pack/plugin/stack/src/yamlRestTest/java/org/elasticsearch/xpack/stack/StackYamlIT.java similarity index 68% rename from x-pack/plugin/stack/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/stack/StackYamlIT.java rename to x-pack/plugin/stack/src/yamlRestTest/java/org/elasticsearch/xpack/stack/StackYamlIT.java index 4db1a6abe6da1..c86d059087530 100644 --- a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/stack/StackYamlIT.java +++ b/x-pack/plugin/stack/src/yamlRestTest/java/org/elasticsearch/xpack/stack/StackYamlIT.java @@ -15,18 +15,32 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; import java.util.Objects; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; - @TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs public class StackYamlIT extends ESClientYamlSuiteTestCase { - private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username")); - private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password")); + private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username", "test_admin")); + private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password", "x-pack-test-password")); + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("x-pack-stack") + .module("x-pack-ilm") + .module("wildcard") + .module("constant-keyword") + .module("ingest-common") + .module("mapper-extras") + .module("data-streams") + .setting("xpack.security.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user(USER, PASS) + .build(); public StackYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); @@ -42,4 +56,9 @@ protected Settings restClientSettings() { String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml similarity index 100% rename from x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml rename to x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java index bb68c7b84da5d..b5064c46c95ae 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java @@ -110,7 +110,7 @@ public void testDeleteWithParamDeletesAutoCreatedDestinationIndex() throws Excep deleteTransform(transformId, false, true); assertFalse(indexExists(transformDest)); - assertFalse(aliasExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); } public void testDeleteWithParamDeletesManuallyCreatedDestinationIndex() throws Exception { @@ -139,7 +139,7 @@ public void testDeleteWithParamDeletesManuallyCreatedDestinationIndex() throws E assertFalse(aliasExists(transformDestAlias)); } - public void testDeleteWithParamDoesNotDeleteManuallySetUpAlias() throws Exception { + public void testDeleteWithManuallyCreatedIndexAndManuallyCreatedAlias() throws Exception { String transformId = "transform-4"; String transformDest = transformId + "_idx"; String transformDestAlias = transformId + "_alias"; @@ -158,31 +158,106 @@ public void testDeleteWithParamDoesNotDeleteManuallySetUpAlias() throws Exceptio assertTrue(indexExists(transformDest)); assertTrue(aliasExists(transformDestAlias)); + deleteTransform(transformId, false, true); + assertFalse(indexExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); + } + + public void testDeleteDestinationIndexIsNoOpWhenNoDestinationIndexExists() throws Exception { + String transformId = "transform-5"; + String transformDest = transformId + "_idx"; + String transformDestAlias = transformId + "_alias"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, transformDestAlias); + + createTransform(transformId, transformDest, transformDestAlias); + assertFalse(indexExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); + + deleteTransform(transformId, false, true); + assertFalse(indexExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); + } + + public void testDeleteWithAliasPointingToManyIndices() throws Exception { + var transformId = "transform-6"; + var transformDest = transformId + "_idx"; + var otherIndex = "some-other-index-6"; + String transformDestAlias = transformId + "_alias"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, otherIndex, transformDestAlias); + + createIndex(transformDest, null, null, "\"" + transformDestAlias + "\": { \"is_write_index\": true }"); + createIndex(otherIndex, null, null, "\"" + transformDestAlias + "\": {}"); + + assertTrue(indexExists(transformDest)); + assertTrue(indexExists(otherIndex)); + assertTrue(aliasExists(transformDestAlias)); + + createTransform(transformId, transformDestAlias, null); + + startTransform(transformId); + waitForTransformCheckpoint(transformId, 1); + + stopTransform(transformId, false); + + assertTrue(indexExists(transformDest)); + assertTrue(indexExists(otherIndex)); + assertTrue(aliasExists(transformDestAlias)); + + deleteTransform(transformId, false, true); + + assertFalse(indexExists(transformDest)); + assertTrue(indexExists(otherIndex)); + assertTrue(aliasExists(transformDestAlias)); + } + + public void testDeleteWithNoWriteIndexThrowsException() throws Exception { + var transformId = "transform-7"; + var transformDest = transformId + "_idx"; + var otherIndex = "some-other-index-7"; + String transformDestAlias = transformId + "_alias"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, otherIndex, transformDestAlias); + + createIndex(transformDest, null, null, "\"" + transformDestAlias + "\": {}"); + + assertTrue(indexExists(transformDest)); + assertTrue(aliasExists(transformDestAlias)); + + createTransform(transformId, transformDestAlias, null); + + createIndex(otherIndex, null, null, "\"" + transformDestAlias + "\": {}"); + assertTrue(indexExists(otherIndex)); + ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, false, true)); assertThat( e.getMessage(), containsString( Strings.format( - "The provided expression [%s] matches an alias, specify the corresponding concrete indices instead.", + "Cannot disambiguate destination index alias [%s]. Alias points to many indices with no clear write alias." + + " Retry with delete_dest_index=false and manually clean up destination index.", transformDestAlias ) ) ); } - public void testDeleteDestinationIndexIsNoOpWhenNoDestinationIndexExists() throws Exception { - String transformId = "transform-5"; - String transformDest = transformId + "_idx"; - String transformDestAlias = transformId + "_alias"; - setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, transformDestAlias); + public void testDeleteWithAlreadyDeletedIndex() throws Exception { + var transformId = "transform-8"; + var transformDest = transformId + "_idx"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest); + + createIndex(transformDest); + + assertTrue(indexExists(transformDest)); + + createTransform(transformId, transformDest, null); + + deleteIndex(transformDest); - createTransform(transformId, transformDest, transformDestAlias); assertFalse(indexExists(transformDest)); - assertFalse(aliasExists(transformDestAlias)); deleteTransform(transformId, false, true); + assertFalse(indexExists(transformDest)); - assertFalse(aliasExists(transformDestAlias)); } private void createTransform(String transformId, String destIndex, String destAlias) throws IOException { diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index 537f50a30b5dd..20ec649f74811 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -412,7 +412,7 @@ protected void updateTransform(String transformId, String update, boolean deferV } updateTransformRequest.setJsonEntity(update); - client().performRequest(updateTransformRequest); + assertOKAndConsume(client().performRequest(updateTransformRequest)); } protected void startTransform(String transformId) throws IOException { diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java index 208da4177fd4c..a4397833d4493 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java @@ -12,6 +12,8 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateAction; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -61,6 +63,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformStats; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; import org.elasticsearch.xpack.transform.LocalStateTransform; +import org.junit.After; import org.junit.Before; import java.io.IOException; @@ -136,6 +139,11 @@ public void setUpNamedXContentRegistryAndIndices() throws Exception { remoteNewDocs = createIndexAndIndexDocs(REMOTE_CLUSTER, "remote_new_index", newRemoteNumShards, timestamp, randomBoolean()); } + @After + public void cleanup() { + client().execute(ResetFeatureStateAction.INSTANCE, new ResetFeatureStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + } + private int createIndexAndIndexDocs(String cluster, String index, int numberOfShards, long timestamp, boolean exposeTimestamp) throws Exception { Client client = client(cluster); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java index cb8580054f8cc..62e2993cf84d3 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java @@ -16,6 +16,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; @@ -25,12 +27,16 @@ import org.elasticsearch.xpack.core.transform.TransformDeprecations; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.GetTransformAction; +import org.elasticsearch.xpack.core.transform.action.PutTransformAction; import org.elasticsearch.xpack.core.transform.action.StartTransformAction; import org.elasticsearch.xpack.core.transform.action.StopTransformAction; import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; +import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests; import org.elasticsearch.xpack.core.transform.utils.TransformConfigVersionUtils; import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase; import org.elasticsearch.xpack.transform.persistence.TransformInternalIndex; @@ -253,4 +259,65 @@ public void testStartReplacesDeprecatedTransformSettings() throws Exception { assertMaxPageSearchSizeInSettings(transformId, expectedMaxPageSearchSize); } + public void testMigratedTransformIndex() { + // create transform + var sourceIndex = "source-index"; + createSourceIndex(sourceIndex); + var transformId = "transform-migrated-system-index"; + + var sourceConfig = new SourceConfig(sourceIndex); + var destConfig = new DestConfig("some-dest-index", null, null); + var config = new TransformConfig( + transformId, + sourceConfig, + destConfig, + null, + null, + null, + PivotConfigTests.randomPivotConfig(), + null, + null, + null, + null, + null, + null, + null + ); + var putTransform = new PutTransformAction.Request(config, true, TimeValue.THIRTY_SECONDS); + assertTrue(client().execute(PutTransformAction.INSTANCE, putTransform).actionGet().isAcknowledged()); + + // simulate migration by reindexing and aliasing + var newSystemIndex = TransformInternalIndexConstants.LATEST_INDEX_NAME + "-reindexed"; + var reindexRequest = new ReindexRequest(); + reindexRequest.setSourceIndices(TransformInternalIndexConstants.LATEST_INDEX_NAME); + reindexRequest.setDestIndex(newSystemIndex); + reindexRequest.setRefresh(true); + client().execute(ReindexAction.INSTANCE, reindexRequest).actionGet(); + + var aliasesRequest = admin().indices().prepareAliases(); + aliasesRequest.removeIndex(TransformInternalIndexConstants.LATEST_INDEX_NAME); + aliasesRequest.addAlias(newSystemIndex, TransformInternalIndexConstants.LATEST_INDEX_NAME); + aliasesRequest.execute().actionGet(); + + // update should succeed + var updateConfig = new TransformConfigUpdate( + sourceConfig, + new DestConfig("some-new-dest-index", null, null), + null, + null, + null, + null, + null, + null + ); + var updateRequest = new UpdateTransformAction.Request(updateConfig, transformId, true, TimeValue.THIRTY_SECONDS); + client().execute(UpdateTransformAction.INSTANCE, updateRequest).actionGet(); + + // verify update succeeded + var getTransformRequest = new GetTransformAction.Request(transformId); + var getTransformResponse = client().execute(GetTransformAction.INSTANCE, getTransformRequest).actionGet(); + var transformConfig = getTransformResponse.getTransformConfigurations().get(0); + assertThat(transformConfig.getDestination().getIndex(), equalTo("some-new-dest-index")); + } + } diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java index cf7441282fa67..86b95e67e356e 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java @@ -78,7 +78,8 @@ public class TransformConfigManagerTests extends TransformSingleNodeTestCase { @Before public void createComponents() { - clusterService = mock(ClusterService.class); + clusterService = mock(); + when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); transformConfigManager = new IndexBasedTransformConfigManager( clusterService, TestIndexNameExpressionResolver.newInstance(), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java index 935ff04c47d85..f5a6f510765b5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java @@ -252,7 +252,7 @@ private static void updateTransformStateAndGetLastCheckpoint( long lastCheckpoint = currentState.v1().getTransformState().getCheckpoint(); // if: the state is stored on the latest index, it does not need an update - if (currentState.v2().getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)) { + if (transformConfigManager.isLatestTransformIndex(currentState.v2().getIndex())) { listener.onResponse(lastCheckpoint); return; } @@ -283,8 +283,7 @@ private static void updateTransformCheckpoint( ActionListener listener ) { transformConfigManager.getTransformCheckpointForUpdate(transformId, lastCheckpoint, ActionListener.wrap(checkpointAndVersion -> { - if (checkpointAndVersion == null - || checkpointAndVersion.v2().getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)) { + if (checkpointAndVersion == null || transformConfigManager.isLatestTransformIndex(checkpointAndVersion.v2().getIndex())) { listener.onResponse(true); return; } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java index 41b683a7965ca..c13043ae2d246 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java @@ -10,9 +10,13 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; import org.elasticsearch.client.internal.Client; @@ -27,6 +31,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -42,6 +47,8 @@ import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.transforms.TransformTask; +import java.util.Objects; + import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.executeWithHeadersAsync; @@ -146,20 +153,31 @@ private void deleteDestinationIndex( TimeValue timeout, ActionListener listener ) { - // <3> Check if the error is "index not found" error. If so, just move on. The index is already deleted. - ActionListener deleteDestIndexListener = ActionListener.wrap(listener::onResponse, e -> { - if (e instanceof IndexNotFoundException) { - listener.onResponse(AcknowledgedResponse.TRUE); - } else { - listener.onFailure(e); - } - }); + getTransformConfig(transformId).andThen((l, r) -> deleteDestinationIndex(r.v1(), parentTaskId, timeout, l)) + .addListener(listener.delegateResponse((l, e) -> { + if (e instanceof IndexNotFoundException) { + l.onResponse(AcknowledgedResponse.TRUE); + } else { + l.onFailure(e); + } + })); + } - // <2> Delete destination index - ActionListener> getTransformConfigurationListener = ActionListener.wrap( - transformConfigAndVersion -> { - TransformConfig config = transformConfigAndVersion.v1(); - String destIndex = config.getDestination().getIndex(); + private SubscribableListener> getTransformConfig(String transformId) { + return SubscribableListener.newForked(l -> transformConfigManager.getTransformConfigurationForUpdate(transformId, l)); + } + + /** + * Delete the destination index. If the Transform is configured to write to an alias, then follow that alias to the concrete index. + */ + private void deleteDestinationIndex( + TransformConfig config, + TaskId parentTaskId, + TimeValue timeout, + ActionListener listener + ) { + SubscribableListener.newForked(l -> resolveDestinationIndex(config, parentTaskId, timeout, l)) + .andThen((l, destIndex) -> { DeleteIndexRequest deleteDestIndexRequest = new DeleteIndexRequest(destIndex); deleteDestIndexRequest.ackTimeout(timeout); deleteDestIndexRequest.setParentTask(parentTaskId); @@ -169,14 +187,57 @@ private void deleteDestinationIndex( client, TransportDeleteIndexAction.TYPE, deleteDestIndexRequest, - deleteDestIndexListener + l ); - }, - listener::onFailure - ); + }) + .addListener(listener); + } + + private void resolveDestinationIndex(TransformConfig config, TaskId parentTaskId, TimeValue timeout, ActionListener listener) { + var destIndex = config.getDestination().getIndex(); + var responseListener = ActionListener.wrap(r -> findDestinationIndexInAliases(r, destIndex, listener), e -> { + if (e instanceof AliasesNotFoundException) { + // no alias == the destIndex is our concrete index + listener.onResponse(destIndex); + } else { + listener.onFailure(e); + } + }); + + GetAliasesRequest request = new GetAliasesRequest(destIndex); + request.setParentTask(parentTaskId); + executeWithHeadersAsync(config.getHeaders(), TRANSFORM_ORIGIN, client, GetAliasesAction.INSTANCE, request, responseListener); + } - // <1> Fetch transform configuration - transformConfigManager.getTransformConfigurationForUpdate(transformId, getTransformConfigurationListener); + private static void findDestinationIndexInAliases(GetAliasesResponse aliases, String destIndex, ActionListener listener) { + var indexToAliases = aliases.getAliases(); + if (indexToAliases.isEmpty()) { + // if the alias list is empty, that means the index is a concrete index + listener.onResponse(destIndex); + } else if (indexToAliases.size() == 1) { + // if there is one value, the alias will treat it as the write index, so it's our destination index + listener.onResponse(indexToAliases.keySet().iterator().next()); + } else { + // if there is more than one index, there may be more than one alias for each index + // we have to search for the alias that matches our destination index name AND is declared the write index for that alias + indexToAliases.entrySet().stream().map(entry -> { + if (entry.getValue().stream().anyMatch(md -> destIndex.equals(md.getAlias()) && Boolean.TRUE.equals(md.writeIndex()))) { + return entry.getKey(); + } else { + return null; + } + }).filter(Objects::nonNull).findFirst().ifPresentOrElse(listener::onResponse, () -> { + listener.onFailure( + new ElasticsearchStatusException( + "Cannot disambiguate destination index alias [" + + destIndex + + "]. Alias points to many indices with no clear write alias. Retry with delete_dest_index=false and manually" + + " clean up destination index.", + RestStatus.CONFLICT + ) + ); + }); + } } @Override diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java index 51e679ff9fe6c..402a8cbe12bd5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/notifications/TransformAuditor.java @@ -52,7 +52,8 @@ public TransformAuditor( nodeName, TransformAuditMessage::new, clusterService, - indexNameExpressionResolver + indexNameExpressionResolver, + clusterService.threadPool().generic() ); clusterService.addListener(event -> { if (event.metadataChanged()) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 50964f5b8cd49..25630fb24772a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -170,7 +170,7 @@ public void updateTransformConfiguration( listener.onFailure(conflictStatusException("Cannot update Transform while the Transform feature is upgrading.")); return; } - if (seqNoPrimaryTermAndIndex.getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_NAME)) { + if (isLatestTransformIndex(seqNoPrimaryTermAndIndex.getIndex())) { // update the config in the same, current index using optimistic concurrency control putTransformConfiguration(transformConfig, DocWriteRequest.OpType.INDEX, seqNoPrimaryTermAndIndex, listener); } else { @@ -180,6 +180,21 @@ public void updateTransformConfiguration( } } + @Override + public boolean isLatestTransformIndex(String indexName) { + if (TransformInternalIndexConstants.LATEST_INDEX_NAME.equals(indexName)) { + return true; + } + + // in some cases, the System Index gets reindexed and LATEST_INDEX_NAME is now an alias pointing to that reindexed index + // this mostly likely happens after the SystemIndexMigrator ran + // we need to check if the LATEST_INDEX_NAME is now an alias and points to the indexName + var metadata = clusterService.state().metadata(); + var indicesForAlias = metadata.aliasedIndices(TransformInternalIndexConstants.LATEST_INDEX_NAME); + var index = metadata.index(indexName); + return index != null && indicesForAlias.contains(index.getIndex()); + } + @Override public void deleteOldTransformConfigurations(String transformId, ActionListener listener) { if (isUpgrading()) { @@ -697,7 +712,7 @@ public void putOrUpdateTransformStoredDoc( // could have been called, see gh#80073 indexRequest.opType(DocWriteRequest.OpType.INDEX); // if on the latest index use optimistic concurrency control in addition - if (seqNoPrimaryTermAndIndex.getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_NAME)) { + if (isLatestTransformIndex(seqNoPrimaryTermAndIndex.getIndex())) { indexRequest.setIfSeqNo(seqNoPrimaryTermAndIndex.getSeqNo()) .setIfPrimaryTerm(seqNoPrimaryTermAndIndex.getPrimaryTerm()); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java index 5ed1a7254268b..ba8ea989cfabf 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc; +import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; import java.util.Collection; import java.util.Collections; @@ -206,4 +207,8 @@ void getTransformStoredDoc( void getTransformStoredDocs(Collection transformIds, TimeValue timeout, ActionListener> listener); void refresh(ActionListener listener); + + default boolean isLatestTransformIndex(String indexName) { + return TransformInternalIndexConstants.LATEST_INDEX_NAME.equals(indexName); + } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java index b9d91287ce45f..3231d705f389c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Tuple; @@ -66,7 +65,6 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; public class TransformUpdaterTests extends ESTestCase { @@ -77,8 +75,7 @@ public class TransformUpdaterTests extends ESTestCase { private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); private TestThreadPool threadPool; private Client client; - private ClusterService clusterService = mock(ClusterService.class); - private TransformAuditor auditor = new MockTransformAuditor(clusterService, mock(IndexNameExpressionResolver.class)); + private TransformAuditor auditor; private final Settings settings = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build(); private final Settings destIndexSettings = new DefaultTransformExtension().getTransformDestinationIndexSettings(); @@ -124,8 +121,7 @@ public void setupClient() { } threadPool = createThreadPool(); client = new MyMockClient(threadPool); - clusterService = mock(ClusterService.class); - auditor = new MockTransformAuditor(clusterService, mock(IndexNameExpressionResolver.class)); + auditor = MockTransformAuditor.createMockAuditor(); } @After diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/notifications/MockTransformAuditor.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/notifications/MockTransformAuditor.java index 1dffd8c20abbf..4eb255b69cfd3 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/notifications/MockTransformAuditor.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/notifications/MockTransformAuditor.java @@ -16,6 +16,8 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.common.notifications.Level; import org.elasticsearch.xpack.core.transform.notifications.TransformAuditMessage; @@ -51,13 +53,16 @@ public static MockTransformAuditor createMockAuditor() { when(state.getMetadata()).thenReturn(metadata); ClusterService clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(state); + ThreadPool threadPool = mock(); + when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(clusterService.threadPool()).thenReturn(threadPool); return new MockTransformAuditor(clusterService, mock(IndexNameExpressionResolver.class)); } private final List expectations; - public MockTransformAuditor(ClusterService clusterService, IndexNameExpressionResolver indexNameResolver) { + private MockTransformAuditor(ClusterService clusterService, IndexNameExpressionResolver indexNameResolver) { super(mock(Client.class), MOCK_NODE_NAME, clusterService, indexNameResolver, true); expectations = new CopyOnWriteArrayList<>(); } diff --git a/x-pack/plugin/vector-tile/build.gradle b/x-pack/plugin/vector-tile/build.gradle index 4ff3240d239e8..a8fa43aeb4db6 100644 --- a/x-pack/plugin/vector-tile/build.gradle +++ b/x-pack/plugin/vector-tile/build.gradle @@ -13,7 +13,7 @@ */ apply plugin: 'elasticsearch.internal-es-plugin' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name = 'vector-tile' @@ -34,12 +34,11 @@ dependencies { // runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714 javaRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0") javaRestTestImplementation("com.google.protobuf:protobuf-java:${versions.protobuf}") -} -testClusters.configureEach { - setting 'xpack.license.self_generated.type', 'trial' - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'false' + clusterModules project(':modules:analysis-common') + clusterModules project(':modules:legacy-geo') + clusterModules project(':modules:lang-painless') + clusterModules project(':test:external-modules:test-error-query') } tasks.named("thirdPartyAudit").configure { diff --git a/x-pack/plugin/vector-tile/qa/build.gradle b/x-pack/plugin/vector-tile/qa/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle b/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle deleted file mode 100644 index 30b032c1cae1a..0000000000000 --- a/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE -apply plugin: 'elasticsearch.legacy-java-rest-test' - -dependencies { - javaRestTestImplementation project(':x-pack:plugin:vector-tile') -} - - -def remoteCluster = testClusters.register('remote') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' -} - -def localCluster = testClusters.register('local') { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' - setting 'cluster.remote.other.seeds', - { "\"${remoteCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE -} - - -def startRemoteCluster = tasks.register("startRemoteCluster", DefaultTestClustersTask.class) { - useCluster remoteCluster - doLast { - "Starting remote cluster before integ tests and integTest cluster is started" - } -} - -tasks.named("javaRestTest").configure { - dependsOn startRemoteCluster - useCluster remoteCluster - useCluster localCluster - doFirst { - nonInputProperties.systemProperty 'tests.local', - "${-> localCluster.get().getAllHttpSocketURI().get(0)}" - nonInputProperties.systemProperty 'tests.remote', - "${-> remoteCluster.get().getAllHttpSocketURI().get(0)}" - } -} diff --git a/x-pack/plugin/vector-tile/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/vectortile/VectorTileCCSIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileCCSIT.java similarity index 84% rename from x-pack/plugin/vector-tile/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/vectortile/VectorTileCCSIT.java rename to x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileCCSIT.java index d58e397704004..e8102630e287f 100644 --- a/x-pack/plugin/vector-tile/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/vectortile/VectorTileCCSIT.java +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileCCSIT.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.vectortile; +package org.elasticsearch.xpack.vectortile; import com.wdtinc.mapbox_vector_tile.VectorTile; @@ -19,20 +19,46 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matchers; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; import java.io.IOException; import java.io.InputStream; public class VectorTileCCSIT extends ESRestTestCase { + public static LocalClusterConfigProvider commonConfig = c -> c.module("vector-tile") + .module("analysis-common") + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "true") + .user("admin", "admin-password", "superuser", false); + + public static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local().name("remote-cluster").apply(commonConfig).build(); + + public static ElasticsearchCluster localCluster = ElasticsearchCluster.local() + .name("local-cluster") + .apply(commonConfig) + .setting("cluster.remote.other.seeds", () -> "\"" + remoteCluster.getTransportEndpoints() + "\"") + .build(); + + @ClassRule + public static RuleChain ruleChain = RuleChain.outerRule(remoteCluster).around(localCluster); + @Override protected Settings restClientSettings() { final String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); } + @Override + protected String getTestRestCluster() { + return localCluster.getHttpAddresses(); + } + private int createIndex(RestClient client, String indexName) throws IOException { final Request createRequest = new Request(HttpPut.METHOD_NAME, indexName); Response response = client.performRequest(createRequest); @@ -113,11 +139,11 @@ private VectorTile.Tile execute(RestClient client, Request mvtRequest) throws IO } private RestClient buildLocalClusterClient() throws IOException { - return buildClient(System.getProperty("tests.local")); + return buildClient(localCluster.getHttpAddresses()); } private RestClient buildRemoteClusterClient() throws IOException { - return buildClient(System.getProperty("tests.remote")); + return buildClient(remoteCluster.getHttpAddresses()); } private RestClient buildClient(final String url) throws IOException { diff --git a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java index 82b6a4382525a..a1db2bbdd7da2 100644 --- a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java @@ -28,11 +28,13 @@ import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.hamcrest.Matchers; import org.junit.AfterClass; import org.junit.Before; +import org.junit.ClassRule; import java.io.IOException; import java.io.InputStream; @@ -47,6 +49,13 @@ */ public class VectorTileRestIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("vector-tile").apply(c -> { + if (Build.current().isSnapshot()) { + c.module("test-error-query"); + } + }).setting("xpack.license.self_generated.type", "trial").build(); + private static final String INDEX_POINTS = "index-points"; private static final String INDEX_POLYGON = "index-polygon"; private static final String INDEX_COLLECTION = "index-collection"; @@ -74,6 +83,11 @@ public void indexDocuments() throws IOException { } } + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + private void indexPoints() throws IOException { final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); Response response = client().performRequest(createRequest); diff --git a/x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..0b26b58eda6a8 --- /dev/null +++ b/x-pack/plugin/watcher/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,16 @@ +ALL-UNNAMED: + - manage_threads + # the original policy has java.net.SocketPermission "*", "accept,connect" + # but a comment stating it was "needed for multiple server implementations used in tests" + # TODO: this is likely not needed, but including here to be on the safe side until + # we can track down whether it's really needed + - inbound_network + - outbound_network + - files: + - relative_path: ".mime.types" + relative_to: "home" + mode: "read" + - relative_path: ".mailcap" + relative_to: "home" + mode: "read" + diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java index 337fc00cc7caf..093959978b0d1 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java @@ -112,8 +112,8 @@ public static void main(String[] args) throws Exception { Node node = new Node( internalNodeEnv, PluginsLoader.createPluginsLoader( - PluginsLoader.loadModulesBundles(internalNodeEnv.modulesFile()), - PluginsLoader.loadPluginsBundles(internalNodeEnv.pluginsFile()), + PluginsLoader.loadModulesBundles(internalNodeEnv.modulesDir()), + PluginsLoader.loadPluginsBundles(internalNodeEnv.pluginsDir()), Map.of() ) ).start() diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java index b37b026b853e2..5c174d1bddef2 100644 --- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -84,6 +84,7 @@ public void testWriteLoadForecastGetsPopulatedDuringRollovers() throws Exception assertAllPreviousForecastsAreClearedAfterRollover(dataStream, metadata); setHasValidLicense(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); @@ -131,6 +132,7 @@ public void testWriteLoadForecastIsOverriddenBySetting() throws Exception { assertAllPreviousForecastsAreClearedAfterRollover(dataStream, metadata); setHasValidLicense(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index d4a85ce859b2b..45c5abdc61fd6 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -19,8 +19,12 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.util.List; import java.util.Objects; import java.util.OptionalDouble; @@ -30,6 +34,9 @@ import static org.elasticsearch.xpack.writeloadforecaster.WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING; class LicensedWriteLoadForecaster implements WriteLoadForecaster { + + private static final Logger logger = LogManager.getLogger(LicensedWriteLoadForecaster.class); + public static final Setting MAX_INDEX_AGE_SETTING = Setting.timeSetting( "write_load_forecaster.max_index_age", TimeValue.timeValueDays(7), @@ -37,23 +44,26 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { Setting.Property.NodeScope, Setting.Property.Dynamic ); - private final BooleanSupplier hasValidLicense; + private final BooleanSupplier hasValidLicenseSupplier; private final ThreadPool threadPool; private volatile TimeValue maxIndexAge; + @SuppressWarnings("unused") // modified via VH_HAS_VALID_LICENSE_FIELD + private volatile boolean hasValidLicense; + LicensedWriteLoadForecaster( - BooleanSupplier hasValidLicense, + BooleanSupplier hasValidLicenseSupplier, ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings ) { - this(hasValidLicense, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); + this(hasValidLicenseSupplier, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(MAX_INDEX_AGE_SETTING, this::setMaxIndexAgeSetting); } // exposed for tests only - LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, ThreadPool threadPool, TimeValue maxIndexAge) { - this.hasValidLicense = hasValidLicense; + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicenseSupplier, ThreadPool threadPool, TimeValue maxIndexAge) { + this.hasValidLicenseSupplier = hasValidLicenseSupplier; this.threadPool = threadPool; this.maxIndexAge = maxIndexAge; } @@ -64,7 +74,7 @@ private void setMaxIndexAgeSetting(TimeValue updatedMaxIndexAge) { @Override public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { - if (hasValidLicense.getAsBoolean() == false) { + if (hasValidLicense == false) { return metadata; } @@ -143,7 +153,7 @@ static OptionalDouble forecastIndexWriteLoad(List indicesWriteLo @Override @SuppressForbidden(reason = "This is the only place where IndexMetadata#getForecastedWriteLoad is allowed to be used") public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { - if (hasValidLicense.getAsBoolean() == false) { + if (hasValidLicense == false) { return OptionalDouble.empty(); } @@ -154,4 +164,29 @@ public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return indexMetadata.getForecastedWriteLoad(); } + + /** + * Used to atomically {@code getAndSet()} the {@link #hasValidLicense} field. This is better than an + * {@link java.util.concurrent.atomic.AtomicBoolean} because it takes one less pointer dereference on each read. + */ + private static final VarHandle VH_HAS_VALID_LICENSE_FIELD; + + static { + try { + VH_HAS_VALID_LICENSE_FIELD = MethodHandles.lookup() + .in(LicensedWriteLoadForecaster.class) + .findVarHandle(LicensedWriteLoadForecaster.class, "hasValidLicense", boolean.class); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @Override + public void refreshLicense() { + final var newValue = hasValidLicenseSupplier.getAsBoolean(); + final var oldValue = (boolean) VH_HAS_VALID_LICENSE_FIELD.getAndSet(this, newValue); + if (newValue != oldValue) { + logger.info("license state changed, now [{}]", newValue ? "valid" : "not valid"); + } + } } diff --git a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index 790af0a201578..162e84b2562c5 100644 --- a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.writeloadforecaster; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataStats; @@ -19,6 +21,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -30,9 +33,12 @@ import java.util.OptionalDouble; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.forecastIndexWriteLoad; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -53,7 +59,13 @@ public void tearDownThreadPool() { public void testWriteLoadForecastIsAddedToWriteIndex() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); - final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); + final AtomicInteger licenseCheckCount = new AtomicInteger(); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> { + licenseCheckCount.incrementAndGet(); + return hasValidLicense.get(); + }, threadPool, maxIndexAge); + + writeLoadForecaster.refreshLicense(); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -95,8 +107,12 @@ public void testWriteLoadForecastIsAddedToWriteIndex() { assertThat(forecastedWriteLoad.isPresent(), is(true)); assertThat(forecastedWriteLoad.getAsDouble(), is(greaterThan(0.0))); + assertThat(licenseCheckCount.get(), equalTo(1)); hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); + assertThat(licenseCheckCount.get(), equalTo(2)); + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); } @@ -136,6 +152,7 @@ public void testUptimeIsUsedToWeightWriteLoad() { metadataBuilder.put(dataStream); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, threadPool, maxIndexAge); + writeLoadForecaster.refreshLicense(); final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( dataStream.getName(), @@ -154,6 +171,7 @@ public void testForecastedWriteLoadIsOverriddenBySetting() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); + writeLoadForecaster.refreshLicense(); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -197,6 +215,7 @@ public void testForecastedWriteLoadIsOverriddenBySetting() { assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(0.6))); hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); @@ -327,4 +346,56 @@ private DataStream createDataStream(String name, List backingIndices) { .setIndexMode(IndexMode.STANDARD) .build(); } + + public void testLicenseStateLogging() { + + final var seenMessages = new ArrayList(); + + final var collectingLoggingAssertion = new MockLog.SeenEventExpectation( + "seen event", + LicensedWriteLoadForecaster.class.getCanonicalName(), + Level.INFO, + "*" + ) { + @Override + public boolean innerMatch(LogEvent event) { + final var message = event.getMessage().getFormattedMessage(); + if (message.startsWith("license state changed, now [")) { + seenMessages.add(message); + return true; + } + + return false; + } + }; + + MockLog.assertThatLogger(() -> { + final var hasValidLicense = new AtomicBoolean(); + final var writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, randomTimeValue()); + assertThat(seenMessages, empty()); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, empty()); + + hasValidLicense.set(true); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]")); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]")); + + hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]", "license state changed, now [not valid]")); + + hasValidLicense.set(true); + ESTestCase.startInParallel(between(1, 10), ignored -> writeLoadForecaster.refreshLicense()); + assertThat( + seenMessages, + contains( + "license state changed, now [valid]", + "license state changed, now [not valid]", + "license state changed, now [valid]" + ) + ); + }, LicensedWriteLoadForecaster.class, collectingLoggingAssertion); + } } diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index 8a67a2c1dde0d..bd00e6750c003 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -27,7 +27,7 @@ tasks.named("yamlRestTest").configure { 'index/10_with_id/Index with ID', 'indices.get_alias/10_basic/Get alias against closed indices' ]; - if (buildParams.isSnapshotBuild() == false) { + if (buildParams.snapshotBuild == false) { blacklist += [ 'synonyms_privileges/10_synonyms_with_privileges/*', 'synonyms_privileges/20_synonyms_no_privileges/*' diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index 0b40828b8e86c..94eeee5ed9298 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -46,6 +46,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.autoconfiguration.enabled", "false") + .systemProperty("es.queryable_built_in_roles_enabled", "false") .user(USER, PASS) .feature(FeatureFlag.TIME_SERIES_MODE) .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java index 261bc567d5c91..74cb057278c4a 100644 --- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java +++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java @@ -152,7 +152,7 @@ public void tearDownMiniKdc() throws IOException, PrivilegedActionException { protected Path getKeytabPath(Environment env) { final Setting setting = KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.getConcreteSettingForNamespace(REALM_NAME); - return env.configFile().resolve(setting.get(settings)); + return env.configDir().resolve(setting.get(settings)); } /** diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 8e74b4fe06098..ac3361cb2a19c 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -41,7 +41,6 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> testDistribution = "DEFAULT" versions = [oldVersion, project.version] numberOfNodes = 3 - systemProperty 'es.queryable_built_in_roles_enabled', 'true' systemProperty 'ingest.geoip.downloader.enabled.default', 'true' //we don't want to hit real service from each test systemProperty 'ingest.geoip.downloader.endpoint.default', 'http://invalid.endpoint' @@ -61,6 +60,10 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> setting 'xpack.security.transport.ssl.key', 'testnode.pem' setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' + if (bwcVersion.before('8.18.0')) { + jvmArgs '-da:org.elasticsearch.index.mapper.MapperService' + jvmArgs '-da:org.elasticsearch.index.mapper.DocumentMapper' + } if (bwcVersion.onOrAfter('7.0.0')) { setting 'xpack.security.authc.realms.file.file1.order', '0' diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index ddef996937883..0156b2120c0f0 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -6,14 +6,18 @@ */ package org.elasticsearch.upgrades; +import org.apache.http.client.methods.HttpPost; import org.apache.http.util.EntityUtils; import org.elasticsearch.Build; +import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.client.Node; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.common.settings.SecureString; @@ -183,19 +187,178 @@ public void testDataStreamValidationDoesNotBreakUpgrade() throws Exception { } public void testUpgradeDataStream() throws Exception { + /* + * This test tests upgrading a "normal" data stream (dataStreamName), and upgrading a data stream that was originally just an + * ordinary index that was converted to a data stream (dataStreamFromNonDataStreamIndices). + */ String dataStreamName = "reindex_test_data_stream"; + String dataStreamFromNonDataStreamIndices = "index_first_reindex_test_data_stream"; int numRollovers = randomIntBetween(0, 5); + boolean hasILMPolicy = minimumTransportVersion().before(TransportVersions.V_8_11_X) || randomBoolean(); + boolean ilmEnabled = hasILMPolicy && randomBoolean(); + + if (ilmEnabled) { + startILM(); + } else { + stopILM(); + } + if (CLUSTER_TYPE == ClusterType.OLD) { - createAndRolloverDataStream(dataStreamName, numRollovers); + createAndRolloverDataStream(dataStreamName, numRollovers, hasILMPolicy, ilmEnabled); + createDataStreamFromNonDataStreamIndices(dataStreamFromNonDataStreamIndices); } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { - upgradeDataStream(dataStreamName, numRollovers); + Map> oldIndicesMetadata = getIndicesMetadata(dataStreamName); + upgradeDataStream(dataStreamName, numRollovers, numRollovers + 1, 0, ilmEnabled); + upgradeDataStream(dataStreamFromNonDataStreamIndices, 0, 1, 0, ilmEnabled); + Map> upgradedIndicesMetadata = getIndicesMetadata(dataStreamName); + + if (ilmEnabled) { + checkILMPhase(dataStreamName, upgradedIndicesMetadata); + } else { + compareIndexMetadata(oldIndicesMetadata, upgradedIndicesMetadata); + } } } - private static void createAndRolloverDataStream(String dataStreamName, int numRollovers) throws IOException { + private void compareIndexMetadata( + Map> oldIndicesMetadata, + Map> upgradedIndicesMetadata + ) { + String oldWriteIndex = getWriteIndexFromDataStreamIndexMetadata(oldIndicesMetadata); + for (Map.Entry> upgradedIndexEntry : upgradedIndicesMetadata.entrySet()) { + String upgradedIndexName = upgradedIndexEntry.getKey(); + if (upgradedIndexName.startsWith(".migrated-")) { + String oldIndexName = "." + upgradedIndexName.substring(".migrated-".length()); + Map oldIndexMetadata = oldIndicesMetadata.get(oldIndexName); + Map upgradedIndexMetadata = upgradedIndexEntry.getValue(); + compareSettings(oldIndexMetadata, upgradedIndexMetadata); + compareMappings((Map) oldIndexMetadata.get("mappings"), (Map) upgradedIndexMetadata.get("mappings")); + assertThat("ILM states did not match", upgradedIndexMetadata.get("ilm"), equalTo(oldIndexMetadata.get("ilm"))); + if (oldIndexName.equals(oldWriteIndex) == false) { // the old write index will have been rolled over by upgrade + assertThat( + "Rollover info did not match", + upgradedIndexMetadata.get("rollover_info"), + equalTo(oldIndexMetadata.get("rollover_info")) + ); + } + assertThat(upgradedIndexMetadata.get("system"), equalTo(oldIndexMetadata.get("system"))); + } + } + } + + @SuppressWarnings("unchecked") + private void checkILMPhase(String dataStreamName, Map> upgradedIndicesMetadata) throws Exception { + var writeIndex = getWriteIndexFromDataStreamIndexMetadata(upgradedIndicesMetadata); + assertBusy(() -> { + + Request request = new Request("GET", dataStreamName + "/_ilm/explain"); + Response response = client().performRequest(request); + Map responseMap = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + response.getEntity().getContent(), + false + ); + Map indices = (Map) responseMap.get("indices"); + for (var index : indices.keySet()) { + if (index.equals(writeIndex) == false) { + Map ilmInfo = (Map) indices.get(index); + assertThat("Index has not moved to cold ILM phase", ilmInfo.get("phase"), equalTo("cold")); + } + } + }, 30, TimeUnit.SECONDS); + } + + private String getWriteIndexFromDataStreamIndexMetadata(Map> indexMetadataForDataStream) { + return indexMetadataForDataStream.entrySet() + .stream() + .sorted((o1, o2) -> Long.compare(getCreationDate(o2.getValue()), getCreationDate(o1.getValue()))) + .map(Map.Entry::getKey) + .findFirst() + .get(); + } + + private void startILM() throws IOException { + setILMInterval(); + var request = new Request("POST", "/_ilm/start"); + assertOK(client().performRequest(request)); + } + + private void stopILM() throws IOException { + var request = new Request("POST", "/_ilm/stop"); + assertOK(client().performRequest(request)); + } + + private void setILMInterval() throws IOException { + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity(""" + { "persistent": {"indices.lifecycle.poll_interval": "1s"} } + """); + assertOK(client().performRequest(request)); + } + + @SuppressWarnings("unchecked") + long getCreationDate(Map indexMetadata) { + return Long.parseLong( + (String) ((Map>) indexMetadata.get("settings")).get("index").get("creation_date") + ); + } + + private void compareSettings(Map oldIndexMetadata, Map upgradedIndexMetadata) { + Map oldIndexSettings = getIndexSettingsFromIndexMetadata(oldIndexMetadata); + Map upgradedIndexSettings = getIndexSettingsFromIndexMetadata(upgradedIndexMetadata); + final Set SETTINGS_TO_CHECK = Set.of( + "lifecycle", + "mode", + "routing", + "hidden", + "number_of_shards", + "creation_date", + "number_of_replicas" + ); + for (String setting : SETTINGS_TO_CHECK) { + assertThat( + "Unexpected value for setting " + setting, + upgradedIndexSettings.get(setting), + equalTo(oldIndexSettings.get(setting)) + ); + } + } + + private void compareMappings(Map oldMappings, Map upgradedMappings) { + boolean ignoreSource = Version.fromString(UPGRADE_FROM_VERSION).before(Version.V_8_18_0); + if (ignoreSource) { + Map doc = (Map) oldMappings.get("_doc"); + if (doc != null) { + Map sourceEntry = (Map) doc.get("_source"); + if (sourceEntry != null && sourceEntry.isEmpty()) { + doc.remove("_source"); + } + assert doc.containsKey("_source") == false; + } + } + assertThat("Mappings did not match", upgradedMappings, equalTo(oldMappings)); + } + + @SuppressWarnings("unchecked") + private Map getIndexSettingsFromIndexMetadata(Map indexMetadata) { + return (Map) ((Map) indexMetadata.get("settings")).get("index"); + } + + private void createAndRolloverDataStream(String dataStreamName, int numRollovers, boolean hasILMPolicy, boolean ilmEnabled) + throws IOException { + if (hasILMPolicy) { + createIlmPolicy(); + } // We want to create a data stream and roll it over several times so that we have several indices to upgrade - final String template = """ + String template = """ { + "settings":{ + "index": { + $ILM_SETTING + "number_of_replicas": 0 + } + }, + $DSL_TEMPLATE "mappings":{ "dynamic_templates": [ { @@ -213,8 +376,7 @@ private static void createAndRolloverDataStream(String dataStreamName, int numRo "type": "date" }, "metricset": { - "type": "keyword", - "time_series_dimension": true + "type": "keyword" }, "k8s": { "properties": { @@ -241,6 +403,19 @@ private static void createAndRolloverDataStream(String dataStreamName, int numRo } } """; + if (hasILMPolicy) { + template = template.replace("$ILM_SETTING", """ + "lifecycle.name": "test-lifecycle-policy", + """); + template = template.replace("$DSL_TEMPLATE", ""); + } else { + template = template.replace("$ILM_SETTING", ""); + template = template.replace("$DSL_TEMPLATE", """ + "lifecycle": { + "data_retention": "7d" + }, + """); + } final String indexTemplate = """ { "index_patterns": ["$PATTERN"], @@ -254,19 +429,212 @@ private static void createAndRolloverDataStream(String dataStreamName, int numRo bulkLoadData(dataStreamName); for (int i = 0; i < numRollovers; i++) { String oldIndexName = rollover(dataStreamName); - if (randomBoolean()) { - closeIndex(oldIndexName); + if (ilmEnabled == false && randomBoolean()) { + closeOrFreezeIndex(oldIndexName); } bulkLoadData(dataStreamName); } } - private void upgradeDataStream(String dataStreamName, int numRolloversOnOldCluster) throws Exception { + // Randomly either closes or freezes the index. If the cluster does not support the _freeze API, then this always closes the index. + private void closeOrFreezeIndex(String indexName) throws IOException { + boolean canFreeze = minimumTransportVersion().before(TransportVersions.V_8_0_0); + if (canFreeze && randomBoolean()) { + final Request freezeRequest = new Request(HttpPost.METHOD_NAME, "/" + indexName + "/_freeze"); + freezeRequest.setOptions( + expectWarnings( + "Frozen indices are deprecated because they provide no benefit given improvements in heap memory utilization. " + + "They will be removed in a future release." + ) + ); + assertOK(client().performRequest(freezeRequest)); + } else { + closeIndex(indexName); + } + } + + private static void createIlmPolicy() throws IOException { + String ilmPolicy = """ + { + "policy": { + "phases": { + "warm": { + "min_age": "1s", + "actions": { + "forcemerge": { + "max_num_segments": 1 + } + } + }, + "cold": { + "actions": { + "set_priority" : { + "priority": 50 + } + } + } + } + } + }"""; + Request putIlmPolicyRequest = new Request("PUT", "_ilm/policy/test-lifecycle-policy"); + putIlmPolicyRequest.setJsonEntity(ilmPolicy); + assertOK(client().performRequest(putIlmPolicyRequest)); + } + + /* + * This returns a Map of index metadata for each index in the data stream, as retrieved from the cluster state. + */ + @SuppressWarnings("unchecked") + private Map> getIndicesMetadata(String dataStreamName) throws IOException { + Request getClusterStateRequest = new Request("GET", "/_cluster/state/metadata/" + dataStreamName); + Response clusterStateResponse = client().performRequest(getClusterStateRequest); + Map clusterState = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + clusterStateResponse.getEntity().getContent(), + false + ); + return ((Map>>) clusterState.get("metadata")).get("indices"); + } + + private void createDataStreamFromNonDataStreamIndices(String dataStreamFromNonDataStreamIndices) throws IOException { + /* + * This method creates an index, creates an alias to that index, and then converts the aliased index into a data stream. This is + * similar to the path that many indices (including system indices) took in versions 7/8. + */ + // First, we create an ordinary index with no @timestamp mapping: + final String templateWithNoTimestamp = """ + { + "mappings":{ + "properties": { + "message": { + "type": "text" + } + } + } + } + """; + // Note that this is not a data stream template: + final String indexTemplate = """ + { + "index_patterns": ["$PATTERN"], + "template": $TEMPLATE + }"""; + var putIndexTemplateRequest = new Request("POST", "/_index_template/reindex_test_data_stream_index_template"); + putIndexTemplateRequest.setJsonEntity( + indexTemplate.replace("$TEMPLATE", templateWithNoTimestamp).replace("$PATTERN", dataStreamFromNonDataStreamIndices + "-*") + ); + String indexName = dataStreamFromNonDataStreamIndices + "-01"; + if (minimumTransportVersion().before(TransportVersions.V_8_0_0)) { + /* + * It is not possible to create a 7.x index template with a type. And you can't create an empty index with a type. But you can + * create the index with a type by posting a document to an index with a type. We do that here so that we test that the type is + * removed when we reindex into 8.x. + */ + String typeName = "test-type"; + Request createIndexRequest = new Request("POST", indexName + "/" + typeName); + createIndexRequest.setJsonEntity(""" + { + "@timestamp": "2099-11-15T13:12:00", + "message": "GET /search HTTP/1.1 200 1070000", + "user": { + "id": "kimchy" + } + }"""); + createIndexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build()); + assertOK(client().performRequest(createIndexRequest)); + } + assertOK(client().performRequest(putIndexTemplateRequest)); + bulkLoadDataMissingTimestamp(indexName); + /* + * Next, we will change the index's mapping to include a @timestamp field since we are going to convert it to a data stream. But + * first we have to flush the translog to disk because adding a @timestamp field will cause errors if it is done before the translog + * is flushed: + */ + assertOK(client().performRequest(new Request("POST", indexName + "/_flush"))); + ensureHealth(indexName, (request -> { + request.addParameter("wait_for_nodes", "3"); + request.addParameter("wait_for_status", "green"); + request.addParameter("timeout", "70s"); + request.addParameter("level", "shards"); + })); + + // Updating the mapping to include @timestamp: + Request updateIndexMappingRequest = new Request("PUT", indexName + "/_mapping"); + updateIndexMappingRequest.setJsonEntity(""" + { + "properties": { + "@timestamp" : { + "type": "date" + }, + "message": { + "type": "text" + } + } + }"""); + assertOK(client().performRequest(updateIndexMappingRequest)); + + // Creating an alias with the same name that the data stream will have: + Request createAliasRequest = new Request("POST", "/_aliases"); + String aliasRequestBody = """ + { + "actions": [ + { + "add": { + "index": "$index", + "alias": "$alias" + } + } + ] + }"""; + createAliasRequest.setJsonEntity( + aliasRequestBody.replace("$index", indexName).replace("$alias", dataStreamFromNonDataStreamIndices) + ); + assertOK(client().performRequest(createAliasRequest)); + + // This is now just an aliased index. We'll convert it into a data stream + final String templateWithTimestamp = """ + { + "mappings":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "message": { + "type": "text" + } + } + } + } + """; + final String dataStreamTemplate = """ + { + "index_patterns": ["$PATTERN"], + "template": $TEMPLATE, + "data_stream": { + } + }"""; + var putDataStreamTemplateRequest = new Request("POST", "/_index_template/reindex_test_data_stream_data_stream_template"); + putDataStreamTemplateRequest.setJsonEntity( + dataStreamTemplate.replace("$TEMPLATE", templateWithTimestamp).replace("$PATTERN", dataStreamFromNonDataStreamIndices) + ); + assertOK(client().performRequest(putDataStreamTemplateRequest)); + Request migrateToDataStreamRequest = new Request("POST", "/_data_stream/_migrate/" + dataStreamFromNonDataStreamIndices); + assertOK(client().performRequest(migrateToDataStreamRequest)); + } + + @SuppressWarnings("unchecked") + private void upgradeDataStream( + String dataStreamName, + int numRolloversOnOldCluster, + int expectedSuccessesCount, + int expectedErrorCount, + boolean ilmEnabled + ) throws Exception { Set indicesNeedingUpgrade = getDataStreamIndices(dataStreamName); final int explicitRolloverOnNewClusterCount = randomIntBetween(0, 2); for (int i = 0; i < explicitRolloverOnNewClusterCount; i++) { String oldIndexName = rollover(dataStreamName); - if (randomBoolean()) { + if (ilmEnabled == false && randomBoolean()) { closeIndex(oldIndexName); } } @@ -329,18 +697,26 @@ private void upgradeDataStream(String dataStreamName, int numRolloversOnOldClust statusResponseMap.get("total_indices_requiring_upgrade"), equalTo(originalWriteIndex + numRolloversOnOldCluster) ); - assertThat(statusResponseString, statusResponseMap.get("successes"), equalTo(numRolloversOnOldCluster + 1)); + assertThat(statusResponseString, statusResponseMap.get("successes"), equalTo(expectedSuccessesCount)); // We expect all the original indices to have been deleted - for (String oldIndex : indicesNeedingUpgrade) { - assertThat(statusResponseString, indexExists(oldIndex), equalTo(false)); + if (expectedErrorCount == 0) { + for (String oldIndex : indicesNeedingUpgrade) { + assertThat(statusResponseString, indexExists(oldIndex), equalTo(false)); + } } assertThat( statusResponseString, getDataStreamIndices(dataStreamName).size(), equalTo(expectedTotalIndicesInDataStream) ); + assertThat(statusResponseString, ((List) statusResponseMap.get("errors")).size(), equalTo(expectedErrorCount)); } }, 60, TimeUnit.SECONDS); + + // Verify it's possible to reindex again after a successful reindex + reindexResponse = upgradeUserClient.performRequest(reindexRequest); + assertOK(reindexResponse); + Request cancelRequest = new Request("POST", "_migration/reindex/" + dataStreamName + "/_cancel"); Response cancelResponse = upgradeUserClient.performRequest(cancelRequest); assertOK(cancelResponse); @@ -394,6 +770,26 @@ private static void bulkLoadData(String dataStreamName) throws IOException { assertOK(response); } + /* + * This bulkloads data, where some documents have no @timestamp field and some do. + */ + private static void bulkLoadDataMissingTimestamp(String dataStreamName) throws IOException { + final String bulk = """ + {"create": {}} + {"metricset": "pod", "k8s": {"pod": {"name": "cat", "network": {"tx": 2001818691, "rx": 802133794}}}} + {"create": {}} + {"metricset": "pod", "k8s": {"pod": {"name": "hamster", "network": {"tx": 2005177954, "rx": 801479970}}}} + {"create": {}} + {"metricset": "pod", "k8s": {"pod": {"name": "cow", "network": {"tx": 2006223737, "rx": 802337279}}}} + {"create": {}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "rat", "network": {"tx": 2012916202, "rx": 803685721}}}} + """; + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + bulkRequest.setJsonEntity(bulk.replace("$now", formatInstant(Instant.now()))); + var response = client().performRequest(bulkRequest); + assertOK(response); + } + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java index f9d28670dab65..79f42244b37c2 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java @@ -65,7 +65,6 @@ protected static void waitForPendingUpgraderTasks() throws Exception { * The purpose of this test is to ensure that when a job is open through a rolling upgrade we upgrade the results * index mappings when it is assigned to an upgraded node even if no other ML endpoint is called after the upgrade */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98560") public void testSnapshotUpgrader() throws Exception { Request adjustLoggingLevels = new Request("PUT", "/_cluster/settings"); adjustLoggingLevels.setJsonEntity(""" @@ -98,6 +97,13 @@ public void testSnapshotUpgrader() throws Exception { @SuppressWarnings("unchecked") private void testSnapshotUpgradeFailsOnMixedCluster() throws Exception { + // TODO the mixed cluster assertions sometimes fail because the code that + // detects the mixed cluster relies on the transport versions being different. + // This assumption does not hold immediately after a version bump and new + // branch being cut as the new branch will have the same transport version + // See https://github.com/elastic/elasticsearch/issues/98560 + + assumeTrue("The mixed cluster is not always detected correctly, see https://github.com/elastic/elasticsearch/issues/98560", false); Map jobs = entityAsMap(getJob(JOB_ID)); String currentSnapshot = ((List) XContentMapValues.extractValue("jobs.model_snapshot_id", jobs)).get(0); @@ -154,7 +160,7 @@ private void testSnapshotUpgrade() throws Exception { List> upgradedSnapshot = (List>) entityAsMap(getModelSnapshots(JOB_ID, snapshotToUpgradeId)) .get("model_snapshots"); - assertThat(upgradedSnapshot, hasSize(1)); + assertThat(upgradedSnapshot.toString(), upgradedSnapshot, hasSize(1)); assertThat(upgradedSnapshot.get(0).get("latest_record_time_stamp"), equalTo(snapshotToUpgrade.get("latest_record_time_stamp"))); // Does the snapshot still work? @@ -273,7 +279,7 @@ private Response buildAndPutJob(String jobId, TimeValue bucketSpan) throws Excep return client().performRequest(request); } - private static List generateData( + static List generateData( long timestamp, TimeValue bucketSpan, int bucketCount, diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlRolloverLegacyIndicesIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlRolloverLegacyIndicesIT.java new file mode 100644 index 0000000000000..7117f6e4eba32 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlRolloverLegacyIndicesIT.java @@ -0,0 +1,322 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.upgrades; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.upgrades.MlJobSnapshotUpgradeIT.generateData; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class MlRolloverLegacyIndicesIT extends AbstractUpgradeTestCase { + + private static final String JOB_ID = "ml-rollover-legacy-job"; + private static final String CUSTOM_INDEX_JOB_ID = "ml-rollover-legacy-custom-job"; + private static final String CUSTOM_RESULTS_INDEX_NAME = "dedicated-results-index"; + private static final String UPGRADED_CLUSTER_JOB_ID = "ml-rollover-upgraded-job"; + private static final String UPGRADED_CUSTOM_INDEX_CLUSTER_JOB_ID = "ml-rollover-upgraded-custom-job"; + private static final int NUM_BUCKETS = 10; + + @BeforeClass + public static void maybeSkip() { + assumeFalse("Skip ML tests on unsupported glibc versions", SKIP_ML_TESTS); + } + + @Override + protected Collection templatesToWaitFor() { + // We shouldn't wait for ML templates during the upgrade - production won't + if (CLUSTER_TYPE != ClusterType.OLD) { + return super.templatesToWaitFor(); + } + return Stream.concat(XPackRestTestConstants.ML_POST_V7120_TEMPLATES.stream(), super.templatesToWaitFor().stream()) + .collect(Collectors.toSet()); + } + + /** + * Test rolling over v7 legacy indices and that the results index aliases are + * updated to point to the new indices. The test covers both the shared and + * custom results indices. + */ + public void testRolloverLegacyIndices() throws Exception { + + switch (CLUSTER_TYPE) { + case OLD: + createAndRunJob(JOB_ID, false); + createAndRunJob(CUSTOM_INDEX_JOB_ID, true); + break; + case MIXED: + break; + case UPGRADED: + assertLegacyIndicesRollover(); + assertAnomalyIndicesRollover(); + assertNotificationsIndexAliasCreated(); + createAndRunJob(UPGRADED_CLUSTER_JOB_ID, false); + closeJob(UPGRADED_CLUSTER_JOB_ID); + createAndRunJob(UPGRADED_CUSTOM_INDEX_CLUSTER_JOB_ID, true); + closeJob(UPGRADED_CUSTOM_INDEX_CLUSTER_JOB_ID); + assertResultsInNewIndex(false); + assertResultsInNewIndex(true); + break; + default: + throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); + } + } + + private void createAndRunJob(String jobId, boolean useCustomIndex) throws IOException { + var resultsIndex = useCustomIndex ? "\"results_index_name\": \"" + CUSTOM_RESULTS_INDEX_NAME + "\"," : ""; + + String jobConfig = Strings.format(""" + { + %s + "analysis_config" : { + "bucket_span": "600s", + "detectors" :[{"function":"metric","field_name":"value","partition_field_name":"series"}] + }, + "data_description" : { + } + }" + """, resultsIndex); + + Request putJob = new Request("PUT", "_ml/anomaly_detectors/" + jobId); + putJob.setJsonEntity(jobConfig); + Response response = client().performRequest(putJob); + assertEquals(200, response.getStatusLine().getStatusCode()); + + Request openJob = new Request("POST", "_ml/anomaly_detectors/" + jobId + "/_open"); + response = client().performRequest(openJob); + assertEquals(200, response.getStatusLine().getStatusCode()); + + TimeValue bucketSpan = TimeValue.timeValueMinutes(10); + long startTime = Instant.now().minus(24L, ChronoUnit.HOURS).toEpochMilli(); + + var dataCounts = entityAsMap( + postData( + jobId, + String.join( + "", + generateData( + startTime, + bucketSpan, + NUM_BUCKETS, + Collections.singletonList("foo"), + (bucketIndex, series) -> bucketIndex == 5 ? 100.0 : 10.0 + ) + ) + ) + ); + assertThat(dataCounts.toString(), (Integer) dataCounts.get("bucket_count"), greaterThan(0)); + flushJob(jobId); + } + + protected Response postData(String jobId, String data) throws IOException { + // Post data is deprecated, so a deprecation warning is possible (depending on the old version) + RequestOptions postDataOptions = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> { + if (warnings.isEmpty()) { + // No warning is OK - it means we hit an old node where post data is not deprecated + return false; + } else if (warnings.size() > 1) { + return true; + } + return warnings.get(0) + .equals( + "Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed" + ) == false; + }).build(); + + Request postDataRequest = new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_data"); + // Post data is deprecated, so expect a deprecation warning + postDataRequest.setOptions(postDataOptions); + postDataRequest.setJsonEntity(data); + return client().performRequest(postDataRequest); + } + + protected void flushJob(String jobId) throws IOException { + client().performRequest(new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_flush")); + } + + private void closeJob(String jobId) throws IOException { + Response closeResponse = client().performRequest(new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_close")); + assertThat(entityAsMap(closeResponse), hasEntry("closed", true)); + } + + @SuppressWarnings("unchecked") + private void assertLegacyIndicesRollover() throws Exception { + if (isOriginalClusterVersionAtLeast(Version.V_8_0_0)) { + // not a legacy index + return; + } + + assertBusy(() -> { + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); // ignore warnings about accessing system index + Request getIndices = new Request("GET", ".ml*"); + getIndices.setOptions(builder); + Response getIndicesResponse = client().performRequest(getIndices); + assertOK(getIndicesResponse); + var asString = EntityUtils.toString(getIndicesResponse.getEntity()); + // legacy -000001 index is rolled over creating -000002 + assertThat(asString, containsString(".ml-state-000002")); + + Request getAliases = new Request("GET", "_alias/.ml*"); + getAliases.setOptions(builder); + Response getAliasesResponse = client().performRequest(getAliases); + + // Check the write alias points to the new index + Map aliasesMap = entityAsMap(getAliasesResponse); + var stateAlias = (Map) aliasesMap.get(".ml-state-000002"); + assertNotNull(stateAlias); + var isHidden = XContentMapValues.extractValue(stateAlias, "aliases", ".ml-state-write", "is_hidden"); + assertEquals(Boolean.TRUE, isHidden); + }); + } + + @SuppressWarnings("unchecked") + private void assertAnomalyIndicesRollover() throws Exception { + if (isOriginalClusterVersionAtLeast(Version.V_8_0_0)) { + // not a legacy index + return; + } + + assertBusy(() -> { + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); // ignore warnings about accessing system index + Request getIndices = new Request("GET", ".ml-anomalies*"); + getIndices.setOptions(builder); + Response getIndicesResponse = client().performRequest(getIndices); + assertOK(getIndicesResponse); + var asString = EntityUtils.toString(getIndicesResponse.getEntity()); + assertThat(asString, containsString(".ml-anomalies-custom-" + CUSTOM_RESULTS_INDEX_NAME)); + assertThat(asString, containsString(".ml-anomalies-custom-" + CUSTOM_RESULTS_INDEX_NAME + "-000001")); + assertThat(asString, containsString(".ml-anomalies-shared")); + assertThat(asString, containsString(".ml-anomalies-shared-000001")); + + Request getAliases = new Request("GET", "_alias/.ml*"); + getAliases.setOptions(builder); + Response getAliasesResponse = client().performRequest(getAliases); + + // Check the write alias points to the new index + Map aliasesResponseMap = entityAsMap(getAliasesResponse); + + String expectedReadAlias = ".ml-anomalies-" + CUSTOM_INDEX_JOB_ID; + String expectedWriteAlias = ".ml-anomalies-.write-" + CUSTOM_INDEX_JOB_ID; + + { + var rolledCustomResultsIndex = (Map) aliasesResponseMap.get( + ".ml-anomalies-custom-" + CUSTOM_RESULTS_INDEX_NAME + "-000001" + ); + assertNotNull(aliasesResponseMap.toString(), rolledCustomResultsIndex); + + var aliases = (Map) rolledCustomResultsIndex.get("aliases"); + assertThat(aliasesResponseMap.toString(), aliases.entrySet(), hasSize(2)); + assertThat(aliasesResponseMap.toString(), aliases.keySet(), containsInAnyOrder(expectedReadAlias, expectedWriteAlias)); + + // Read alias + var isHidden = XContentMapValues.extractValue(rolledCustomResultsIndex, "aliases", expectedReadAlias, "is_hidden"); + assertEquals(Boolean.TRUE, isHidden); + var isWrite = XContentMapValues.extractValue(rolledCustomResultsIndex, "aliases", expectedReadAlias, "is_write_index"); + assertNull(isWrite); // not a write index + var filter = XContentMapValues.extractValue(rolledCustomResultsIndex, "aliases", expectedReadAlias, "filter"); + assertNotNull(filter); + + // Write alias + isHidden = XContentMapValues.extractValue(rolledCustomResultsIndex, "aliases", expectedWriteAlias, "is_hidden"); + assertEquals(Boolean.TRUE, isHidden); + isWrite = XContentMapValues.extractValue(rolledCustomResultsIndex, "aliases", expectedWriteAlias, "is_write_index"); + assertEquals(Boolean.TRUE, isWrite); + filter = XContentMapValues.extractValue(rolledCustomResultsIndex, "aliases", expectedReadAlias, "filter"); + assertNotNull(filter); + } + + { + var olcustomResultsIndex = (Map) aliasesResponseMap.get( + ".ml-anomalies-custom-" + CUSTOM_RESULTS_INDEX_NAME + ); + assertNotNull(aliasesResponseMap.toString(), olcustomResultsIndex); + var aliases = (Map) olcustomResultsIndex.get("aliases"); + assertThat(aliasesResponseMap.toString(), aliases.entrySet(), hasSize(1)); + assertThat(aliasesResponseMap.toString(), aliases.keySet(), containsInAnyOrder(expectedReadAlias)); + + // Read alias + var isHidden = XContentMapValues.extractValue(olcustomResultsIndex, "aliases", expectedReadAlias, "is_hidden"); + assertEquals(Boolean.TRUE, isHidden); + var isWrite = XContentMapValues.extractValue(olcustomResultsIndex, "aliases", expectedReadAlias, "is_write_index"); + assertNull(isWrite); // not a write index + var filter = XContentMapValues.extractValue(olcustomResultsIndex, "aliases", expectedReadAlias, "filter"); + assertNotNull(filter); + } + }); + } + + @SuppressWarnings("unchecked") + public void assertResultsInNewIndex(boolean checkCustomIndex) throws Exception { + if (isOriginalClusterVersionAtLeast(Version.V_8_0_0)) { + // not a legacy index + return; + } + + var searchUrl = checkCustomIndex + ? ".ml-anomalies-custom-" + CUSTOM_RESULTS_INDEX_NAME + "-000001/_search" + : ".ml-anomalies-shared-000001/_search"; + + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setWarningsHandler(WarningsHandler.PERMISSIVE); // ignore warnings about accessing hidden index + Request getIndices = new Request("GET", searchUrl); + getIndices.setOptions(builder); + Response searchResponse = client().performRequest(getIndices); + assertOK(searchResponse); + + final Map responseMap = responseAsMap(searchResponse); + Map hits = ((Map) responseMap.get("hits")); + assertEquals(responseMap.toString(), NUM_BUCKETS, ((List) hits.get("hits")).size()); + } + + @SuppressWarnings("unchecked") + private void assertNotificationsIndexAliasCreated() throws Exception { + assertBusy(() -> { + Request getMappings = new Request("GET", "_alias/.ml-notifications-write"); + Response response = client().performRequest(getMappings); + Map responseMap = entityAsMap(response); + assertThat(responseMap.entrySet(), hasSize(1)); + var aliases = (Map) responseMap.get(".ml-notifications-000002"); + assertThat(aliases.entrySet(), hasSize(1)); + var allAliases = (Map) aliases.get("aliases"); + var writeAlias = (Map) allAliases.get(".ml-notifications-write"); + + assertThat(writeAlias, hasEntry("is_hidden", Boolean.TRUE)); + var isWriteIndex = (Boolean) writeAlias.get("is_write_index"); + assertThat(isWriteIndex, anyOf(is(Boolean.TRUE), nullValue())); + }); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java new file mode 100644 index 0000000000000..2482a4a3adbb1 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.upgrades; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.test.SecuritySettingsSourceField; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class SystemIndicesUpgradeIT extends AbstractUpgradeTestCase { + private static final Logger log = LogManager.getLogger(SystemIndicesUpgradeIT.class); + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( + "test_user", + new SecureString(SecuritySettingsSourceField.TEST_PASSWORD) + ); + + @BeforeClass + public static void avoidBugIn8_0_1() { + assumeTrue("https://github.com/elastic/elasticsearch/issues/125167", isOriginalClusterVersionAtLeast(Version.V_8_1_0)); + } + + @Override + protected Settings restAdminSettings() { + // Note that we are both superuser here and provide a product origin + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE) + .put(ThreadContext.PREFIX + "." + Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, "fleet") + .build(); + } + + public void testUpgradeSystemIndexAndDataStream() throws Exception { + String dataStreamName = ".fleet-actions-results"; + String indexName = ".fleet-actions"; + if (CLUSTER_TYPE == AbstractUpgradeTestCase.ClusterType.OLD) { + addDataTo(dataStreamName); + addDataTo(indexName); + verifyDataStream(dataStreamName); + verifyAccessToIndex(dataStreamName); + verifyAccessToIndex(indexName); + } else if (CLUSTER_TYPE == AbstractUpgradeTestCase.ClusterType.UPGRADED) { + upgradeSystemIndices(); + verifyDataStream(dataStreamName); + verifyIndex(indexName); + verifyAccessToIndex(dataStreamName); + verifyAccessToIndex(indexName); + } + } + + private void verifyDataStream(String dataStreamName) throws IOException { + Map>> metadata = getMetadata(dataStreamName); + assertThat(getProperty(metadata, List.of("data_stream", "data_stream", dataStreamName, "system")), equalTo("true")); + + Map> upgradedIndicesMetadata = metadata.get("indices"); + for (Map.Entry> indexEntry : upgradedIndicesMetadata.entrySet()) { + Map indexProperties = indexEntry.getValue(); + verifySystemIndexProperties(indexProperties); + } + } + + private static void verifyAccessToIndex(String aliasOrDataStreamName) throws IOException { + Request fleetCountRequest = new Request("GET", aliasOrDataStreamName + "/_count"); + Response fleetCountResponse = adminClient().performRequest(fleetCountRequest); + assertOK(fleetCountResponse); + assertThat( + XContentHelper.convertToMap(JsonXContent.jsonXContent, fleetCountResponse.getEntity().getContent(), false).get("count"), + equalTo(1) + ); + } + + private void addDataTo(String indexName) throws IOException { + Request request = new Request("POST", indexName + "/_doc"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"@timestamp\": 0}"); + assertOK(adminClient().performRequest(request)); + } + + private void verifyIndex(String indexName) throws IOException { + Map> indexMetadata = getIndexMetadata(indexName); + assertThat(indexMetadata, aMapWithSize(1)); + Map indexProperties = indexMetadata.values().iterator().next(); + verifySystemIndexProperties(indexProperties); + } + + private static void verifySystemIndexProperties(Map indexProperties) { + assertThat(getProperty(indexProperties, List.of("system")), equalTo("true")); + } + + @SuppressWarnings("unchecked") + private static String getProperty(Map properties, List propertyPath) { + for (int i = 0; i < propertyPath.size() - 1; i++) { + Object o = properties.get(propertyPath.get(i)); + assertThat(o, instanceOf(Map.class)); + properties = (Map) o; + } + return String.valueOf(properties.get(propertyPath.get(propertyPath.size() - 1))); + } + + private void upgradeSystemIndices() throws Exception { + String upgradeUser = "upgrade_user"; + String upgradeUserPassword = "x-pack-test-password"; + createRole("upgrade_role"); + createUser(upgradeUser, upgradeUserPassword, "upgrade_role"); + + try (RestClient upgradeUserClient = getClient(upgradeUser, upgradeUserPassword)) { + boolean upgradeRequired = Version.fromString(UPGRADE_FROM_VERSION).before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION); + String expectedStatus = (upgradeRequired) ? "MIGRATION_NEEDED" : "NO_MIGRATION_NEEDED"; + + assertThat( + XContentHelper.convertToMap( + JsonXContent.jsonXContent, + upgradeUserClient.performRequest(new Request("GET", "/_migration/system_features")).getEntity().getContent(), + false + ).get("migration_status"), + equalTo(expectedStatus) + ); + + if (upgradeRequired) { + Request upgradeRequest = new Request("POST", "/_migration/system_features"); + Response upgradeResponse = upgradeUserClient.performRequest(upgradeRequest); + assertOK(upgradeResponse); + assertBusy(() -> { + Response featureResponse = upgradeUserClient.performRequest(new Request("GET", "/_migration/system_features")); + assertThat( + XContentHelper.convertToMap(JsonXContent.jsonXContent, featureResponse.getEntity().getContent(), false) + .get("migration_status"), + equalTo("NO_MIGRATION_NEEDED") + ); + }, 30, TimeUnit.SECONDS); + } + } + } + + private void createUser(String name, String password, String role) throws IOException { + Request request = new Request("PUT", "/_security/user/" + name); + request.setJsonEntity("{ \"password\": \"" + password + "\", \"roles\": [ \"" + role + "\"] }"); + assertOK(adminClient().performRequest(request)); + } + + private void createRole(String name) throws IOException { + Request request = new Request("PUT", "/_security/role/" + name); + request.setJsonEntity( + "{ \"cluster\": [\"cluster:admin/migration/post_system_feature\", \"cluster:admin/migration/get_system_feature\"] }" + ); + assertOK(adminClient().performRequest(request)); + } + + private RestClient getClient(String user, String passwd) throws IOException { + RestClientBuilder builder = RestClient.builder(adminClient().getNodes().toArray(new Node[0])); + String token = basicAuthHeaderValue(user, new SecureString(passwd.toCharArray())); + configureClient(builder, Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build()); + builder.setStrictDeprecationMode(true); + return builder.build(); + } + + private Map> getIndexMetadata(String aliasName) throws IOException { + Map>> metadata = getMetadata(aliasName); + return metadata.get("indices"); + } + + @SuppressWarnings("unchecked") + private static Map>> getMetadata(String dataStreamOrAlias) throws IOException { + Request getClusterStateRequest = new Request("GET", "/_cluster/state/metadata/" + dataStreamOrAlias); + Response clusterStateResponse = client().performRequest(getClusterStateRequest); + Map clusterState = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + clusterStateResponse.getEntity().getContent(), + false + ); + return (Map>>) clusterState.get("metadata"); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml index a8501332b973a..a26d67e78089f 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml @@ -7,6 +7,9 @@ setup: timeout: 70s --- "Get start, stop, upgrade and delete old cluster batch transform": + - skip: + awaits_fix: "https://github.com/elastic/ml-team/issues/1522" + reason: "Transform system index migration is broken" # Simple and complex OLD transforms - do: transform.get_transform: @@ -94,6 +97,9 @@ setup: --- "Get start, stop mixed cluster batch transform": # Simple and complex Mixed cluster transforms + - skip: + awaits_fix: "https://github.com/elastic/ml-team/issues/1522" + reason: "Transform system index migration is broken" - do: transform.get_transform: transform_id: "mixed-simple-transform" @@ -222,6 +228,9 @@ setup: transform_id: "old-simple-continuous-transform" --- "Test GET, mixed continuous transforms": + - skip: + awaits_fix: "https://github.com/elastic/ml-team/issues/1522" + reason: "Transform system index migration is broken" - do: transform.get_transform: transform_id: "mixed-simple-continuous-transform" @@ -286,8 +295,13 @@ setup: --- "Test index mappings for latest internal index and audit index": + - skip: + awaits_fix: "https://github.com/elastic/ml-team/issues/1522" + reason: "Transform system index migration is broken" - skip: features: warnings + - requires: + test_runner_features: warnings_regex - do: transform.put_transform: transform_id: "upgraded-simple-transform" @@ -304,8 +318,8 @@ setup: - match: { acknowledged: true } - do: - warnings: - - "this request accesses system indices: [.transform-internal-007], but in a future major version, direct access to system indices will be prevented by default" + warnings_regex: + - "this request accesses system indices: \\[\\.transform-internal-\\d{3}(?:-reindexed-for-\\d{1,2})?], but in a future major version, direct access to system indices will be prevented by default" indices.get_mapping: index: .transform-internal-007 - match: { \.transform-internal-007.mappings.dynamic: "false" } diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java index 5d8684bf32f89..3dbc4c1bf186f 100644 --- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java +++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java @@ -36,7 +36,7 @@ public class ExampleSecurityExtension implements SecurityExtension { static { - final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled")); + final boolean useEntitlements = true; if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) { // check that the extension's policy works. AccessController.doPrivileged((PrivilegedAction) () -> { diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHashTests.java b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHashTests.java index 95c3fd4fde916..a5330d3daf92f 100644 --- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHashTests.java +++ b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/AutoConfigGenerateElasticPasswordHashTests.java @@ -97,18 +97,18 @@ protected Environment createEnv(OptionSet options, ProcessInfo processInfo) thro public void testSuccessfullyGenerateAndStoreHash() throws Exception { execute(); assertThat(terminal.getOutput(), hasLength(20)); - KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(env.configFile()); + KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(env.configDir()); assertNotNull(keyStoreWrapper); keyStoreWrapper.decrypt(new char[0]); assertThat(keyStoreWrapper.getSettingNames(), containsInAnyOrder(AUTOCONFIG_ELASTIC_PASSWORD_HASH.getKey(), "keystore.seed")); } public void testExistingKeystoreWithWrongPassword() throws Exception { - KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(env.configFile()); + KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.load(env.configDir()); assertNotNull(keyStoreWrapper); keyStoreWrapper.decrypt(new char[0]); // set a random password so that we fail to decrypt it in GenerateElasticPasswordHash#execute - keyStoreWrapper.save(env.configFile(), randomAlphaOfLength(16).toCharArray()); + keyStoreWrapper.save(env.configDir(), randomAlphaOfLength(16).toCharArray()); UserException e = expectThrows(UserException.class, this::execute); assertThat(e.getMessage(), equalTo("Failed to generate a password for the elastic user")); assertThat(terminal.getOutput(), is(emptyString())); diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java index 1a0f098b45bde..93281e3453e5c 100644 --- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java @@ -17,20 +17,15 @@ public final class XPackRestTestConstants { public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] { HISTORY_TEMPLATE_NAME_NO_ILM }; // ML constants: - public static final String ML_META_INDEX_NAME = ".ml-meta"; - public static final String CONFIG_INDEX = ".ml-config"; public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; public static final String STATE_INDEX_PREFIX = ".ml-state"; - public static final String RESULTS_INDEX_DEFAULT = "shared"; public static final List ML_POST_V7120_TEMPLATES = List.of(STATE_INDEX_PREFIX, RESULTS_INDEX_PREFIX); // Transform constants: public static final String TRANSFORM_TASK_NAME = "data_frame/transforms"; public static final String TRANSFORM_INTERNAL_INDEX_PREFIX = ".transform-internal-"; - public static final String TRANSFORM_NOTIFICATIONS_INDEX_PREFIX = ".transform-notifications-"; public static final String TRANSFORM_INTERNAL_INDEX_PREFIX_DEPRECATED = ".data-frame-internal-"; - public static final String TRANSFORM_NOTIFICATIONS_INDEX_PREFIX_DEPRECATED = ".data-frame-notifications-"; private XPackRestTestConstants() {} } diff --git a/x-pack/rest-resources-zip/build.gradle b/x-pack/rest-resources-zip/build.gradle index 00753de6a6dae..a613d91d8e9fe 100644 --- a/x-pack/rest-resources-zip/build.gradle +++ b/x-pack/rest-resources-zip/build.gradle @@ -26,7 +26,7 @@ dependencies { freeCompatTests project(path: ':rest-api-spec', configuration: 'restCompatTests') platinumTests project(path: ':x-pack:plugin', configuration: 'restXpackTests') platinumTests project(path: ':x-pack:plugin:eql:qa:rest', configuration: 'restXpackTests') - platinumTests project(path: ':x-pack:plugin:ent-search:qa:rest', configuration: 'restXpackTests') + platinumTests project(path: ':x-pack:plugin:ent-search', configuration: 'restXpackTests') platinumCompatTests project(path: ':x-pack:plugin', configuration: 'restCompatTests') platinumCompatTests project(path: ':x-pack:plugin:eql:qa:rest', configuration: 'restCompatTests') }